Attachment #8451706: update mozprocess to the latest for mozharness for bug #1026710

View | Details | Raw Unified | Return to bug 1026710
Collapse All | Expand All

(-)a/mozprocess/processhandler.py (-42 / +104 lines)
Line     Link Here 
 Lines 34-52   class ProcessHandlerMixin(object): Link Here 
34
    A class for launching and manipulating local processes.
34
    A class for launching and manipulating local processes.
35
35
36
    :param cmd: command to run. May be a string or a list. If specified as a list, the first element will be interpreted as the command, and all additional elements will be interpreted as arguments to that command.
36
    :param cmd: command to run. May be a string or a list. If specified as a list, the first element will be interpreted as the command, and all additional elements will be interpreted as arguments to that command.
37
    :param args: list of arguments to pass to the command (defaults to None). Must not be set when `cmd` is specified as a list.
37
    :param args: list of arguments to pass to the command (defaults to None). Must not be set when `cmd` is specified as a list.
38
    :param cwd: working directory for command (defaults to None).
38
    :param cwd: working directory for command (defaults to None).
39
    :param env: is the environment to use for the process (defaults to os.environ).
39
    :param env: is the environment to use for the process (defaults to os.environ).
40
    :param ignore_children: causes system to ignore child processes when True, defaults to False (which tracks child processes).
40
    :param ignore_children: causes system to ignore child processes when True, defaults to False (which tracks child processes).
41
    :param kill_on_timeout: when True, the process will be killed when a timeout is reached. When False, the caller is responsible for killing the process. Failure to do so could cause a call to wait() to hang indefinitely. (Defaults to True.)
41
    :param kill_on_timeout: when True, the process will be killed when a timeout is reached. When False, the caller is responsible for killing the process. Failure to do so could cause a call to wait() to hang indefinitely. (Defaults to True.)
42
    :param processOutputLine: function to be called for each line of output produced by the process (defaults to None).
42
    :param processOutputLine: function or list of functions to be called for each line of output produced by the process (defaults to None).
43
    :param onTimeout: function to be called when the process times out.
43
    :param onTimeout: function or list of functions to be called when the process times out.
44
    :param onFinish: function to be called when the process terminates normally without timing out.
44
    :param onFinish: function or list of functions to be called when the process terminates normally without timing out.
45
    :param kwargs: additional keyword args to pass directly into Popen.
45
    :param kwargs: additional keyword args to pass directly into Popen.
46
46
47
    NOTE: Child processes will be tracked by default.  If for any reason
47
    NOTE: Child processes will be tracked by default.  If for any reason
48
    we are unable to track child processes and ignore_children is set to False,
48
    we are unable to track child processes and ignore_children is set to False,
49
    then we will fall back to only tracking the root process.  The fallback
49
    then we will fall back to only tracking the root process.  The fallback
50
    will be logged.
50
    will be logged.
51
    """
51
    """
52
52
 Lines 95-173   class ProcessHandlerMixin(object): Link Here 
95
                                          shell, cwd, env,
95
                                          shell, cwd, env,
96
                                          universal_newlines, startupinfo, creationflags)
96
                                          universal_newlines, startupinfo, creationflags)
97
            except OSError, e:
97
            except OSError, e:
98
                print >> sys.stderr, args
98
                print >> sys.stderr, args
99
                raise
99
                raise
100
100
101
        def __del__(self, _maxint=sys.maxint):
101
        def __del__(self, _maxint=sys.maxint):
102
            if isWin:
102
            if isWin:
103
                if self._handle:
103
                handle = getattr(self, '_handle', None)
104
                if handle:
104
                    if hasattr(self, '_internal_poll'):
105
                    if hasattr(self, '_internal_poll'):
105
                        self._internal_poll(_deadstate=_maxint)
106
                        self._internal_poll(_deadstate=_maxint)
106
                    else:
107
                    else:
107
                        self.poll(_deadstate=sys.maxint)
108
                        self.poll(_deadstate=sys.maxint)
108
                if self._handle or self._job or self._io_port:
109
                if handle or self._job or self._io_port:
109
                    self._cleanup()
110
                    self._cleanup()
110
            else:
111
            else:
111
                subprocess.Popen.__del__(self)
112
                subprocess.Popen.__del__(self)
112
113
113
        def kill(self, sig=None):
114
        def kill(self, sig=None):
114
            self.returncode = 0
115
            if isWin:
115
            if isWin:
116
                if not self._ignore_children and self._handle and self._job:
116
                if not self._ignore_children and self._handle and self._job:
117
                    winprocess.TerminateJobObject(self._job, winprocess.ERROR_CONTROL_C_EXIT)
117
                    winprocess.TerminateJobObject(self._job, winprocess.ERROR_CONTROL_C_EXIT)
118
                    self.returncode = winprocess.GetExitCodeProcess(self._handle)
118
                    self.returncode = winprocess.GetExitCodeProcess(self._handle)
119
                elif self._handle:
119
                elif self._handle:
120
                    err = None
120
                    err = None
121
                    try:
121
                    try:
122
                        winprocess.TerminateProcess(self._handle, winprocess.ERROR_CONTROL_C_EXIT)
122
                        winprocess.TerminateProcess(self._handle, winprocess.ERROR_CONTROL_C_EXIT)
123
                    except:
123
                    except:
124
                        err = "Could not terminate process"
124
                        err = "Could not terminate process"
125
                    self.returncode = winprocess.GetExitCodeProcess(self._handle)
125
                    winprocess.GetExitCodeProcess(self._handle)
126
                    self._cleanup()
126
                    self._cleanup()
127
                    if err is not None:
127
                    if err is not None:
128
                        raise OSError(err)
128
                        raise OSError(err)
129
            else:
129
            else:
130
                sig = sig or signal.SIGKILL
130
                sig = sig or signal.SIGKILL
131
                if not self._ignore_children:
131
                if not self._ignore_children:
132
                    try:
132
                    try:
133
                        os.killpg(self.pid, sig)
133
                        os.killpg(self.pid, sig)
134
                    except BaseException, e:
134
                    except BaseException, e:
135
                        if getattr(e, "errno", None) != 3:
135
                        if getattr(e, "errno", None) != 3:
136
                            # Error 3 is "no such process", which is ok
136
                            # Error 3 is "no such process", which is ok
137
                            print >> sys.stdout, "Could not kill process, could not find pid: %s, assuming it's already dead" % self.pid
137
                            print >> sys.stdout, "Could not kill process, could not find pid: %s, assuming it's already dead" % self.pid
138
                else:
138
                else:
139
                    os.kill(self.pid, sig)
139
                    os.kill(self.pid, sig)
140
                self.returncode = -sig
141
140
141
            self.returncode = self.wait()
142
            self._cleanup()
142
            self._cleanup()
143
            return self.returncode
143
            return self.returncode
144
144
145
        def poll(self):
146
            """ Popen.poll
147
                Check if child process has terminated. Set and return returncode attribute.
148
            """
149
            # If we have a handle, the process is alive
150
            if isWin and getattr(self, '_handle', None):
151
                return None
152
153
            return subprocess.Popen.poll(self)
154
145
        def wait(self):
155
        def wait(self):
146
            """ Popen.wait
156
            """ Popen.wait
147
                Called to wait for a running process to shut down and return
157
                Called to wait for a running process to shut down and return
148
                its exit code
158
                its exit code
149
                Returns the main process's exit code
159
                Returns the main process's exit code
150
            """
160
            """
151
            # This call will be different for each OS
161
            # This call will be different for each OS
152
            self.returncode = self._wait()
162
            self.returncode = self._wait()
153
            self._cleanup()
163
            self._cleanup()
154
            return self.returncode
164
            return self.returncode
155
165
156
        """ Private Members of Process class """
166
        """ Private Members of Process class """
157
167
158
        if isWin:
168
        if isWin:
159
            # Redefine the execute child so that we can track process groups
169
            # Redefine the execute child so that we can track process groups
160
            def _execute_child(self, args, executable, preexec_fn, close_fds,
170
            def _execute_child(self, *args_tuple):
161
                               cwd, env, universal_newlines, startupinfo,
171
                # workaround for bug 950894
162
                               creationflags, shell,
172
                if sys.hexversion < 0x02070600: # prior to 2.7.6
163
                               p2cread, p2cwrite,
173
                    (args, executable, preexec_fn, close_fds,
164
                               c2pread, c2pwrite,
174
                     cwd, env, universal_newlines, startupinfo,
165
                               errread, errwrite):
175
                     creationflags, shell,
176
                     p2cread, p2cwrite,
177
                     c2pread, c2pwrite,
178
                     errread, errwrite) = args_tuple
179
                    to_close = set()
180
                else: # 2.7.6 and later
181
                    (args, executable, preexec_fn, close_fds,
182
                     cwd, env, universal_newlines, startupinfo,
183
                     creationflags, shell, to_close,
184
                     p2cread, p2cwrite,
185
                     c2pread, c2pwrite,
186
                     errread, errwrite) = args_tuple
166
                if not isinstance(args, basestring):
187
                if not isinstance(args, basestring):
167
                    args = subprocess.list2cmdline(args)
188
                    args = subprocess.list2cmdline(args)
168
189
169
                # Always or in the create new process group
190
                # Always or in the create new process group
170
                creationflags |= winprocess.CREATE_NEW_PROCESS_GROUP
191
                creationflags |= winprocess.CREATE_NEW_PROCESS_GROUP
171
192
172
                if startupinfo is None:
193
                if startupinfo is None:
173
                    startupinfo = winprocess.STARTUPINFO()
194
                    startupinfo = winprocess.STARTUPINFO()
 Lines 536-552   falling back to not using job objects fo Link Here 
536
                            return status >> 8
557
                            return status >> 8
537
                        return -status
558
                        return -status
538
                    except OSError, e:
559
                    except OSError, e:
539
                        if getattr(e, "errno", None) != 10:
560
                        if getattr(e, "errno", None) != 10:
540
                            # Error 10 is "no child process", which could indicate normal
561
                            # Error 10 is "no child process", which could indicate normal
541
                            # close
562
                            # close
542
                            print >> sys.stderr, "Encountered error waiting for pid to close: %s" % e
563
                            print >> sys.stderr, "Encountered error waiting for pid to close: %s" % e
543
                            raise
564
                            raise
544
                        return 0
565
566
                        return self.returncode
545
567
546
                else:
568
                else:
547
                    # For non-group wait, call base class
569
                    # For non-group wait, call base class
548
                    subprocess.Popen.wait(self)
570
                    subprocess.Popen.wait(self)
549
                    return self.returncode
571
                    return self.returncode
550
572
551
            def _cleanup(self):
573
            def _cleanup(self):
552
                pass
574
                pass
 Lines 583-600   falling back to not using job objects fo Link Here 
583
        self.outThread = None
605
        self.outThread = None
584
        self.read_buffer = ''
606
        self.read_buffer = ''
585
607
586
        if env is None:
608
        if env is None:
587
            env = os.environ.copy()
609
            env = os.environ.copy()
588
        self.env = env
610
        self.env = env
589
611
590
        # handlers
612
        # handlers
613
        if callable(processOutputLine):
614
            processOutputLine = [processOutputLine]
591
        self.processOutputLineHandlers = list(processOutputLine)
615
        self.processOutputLineHandlers = list(processOutputLine)
616
        if callable(onTimeout):
617
            onTimeout = [onTimeout]
592
        self.onTimeoutHandlers = list(onTimeout)
618
        self.onTimeoutHandlers = list(onTimeout)
619
        if callable(onFinish):
620
            onFinish = [onFinish]
593
        self.onFinishHandlers = list(onFinish)
621
        self.onFinishHandlers = list(onFinish)
594
622
595
        # It is common for people to pass in the entire array with the cmd and
623
        # It is common for people to pass in the entire array with the cmd and
596
        # the args together since this is how Popen uses it.  Allow for that.
624
        # the args together since this is how Popen uses it.  Allow for that.
597
        if isinstance(self.cmd, list):
625
        if isinstance(self.cmd, list):
598
            if self.args != None:
626
            if self.args != None:
599
                raise TypeError("cmd and args must not both be lists")
627
                raise TypeError("cmd and args must not both be lists")
600
            (self.cmd, self.args) = (self.cmd[0], self.cmd[1:])
628
            (self.cmd, self.args) = (self.cmd[0], self.cmd[1:])
 Lines 652-671   falling back to not using job objects fo Link Here 
652
680
653
        Note that this does not manage any state, save any output etc,
681
        Note that this does not manage any state, save any output etc,
654
        it immediately kills the process.
682
        it immediately kills the process.
655
683
656
        :param sig: Signal used to kill the process, defaults to SIGKILL
684
        :param sig: Signal used to kill the process, defaults to SIGKILL
657
                    (has no effect on Windows)
685
                    (has no effect on Windows)
658
        """
686
        """
659
        try:
687
        try:
660
            return self.proc.kill(sig=sig)
688
            self.proc.kill(sig=sig)
689
690
            # When we kill the the managed process we also have to wait for the
691
            # outThread to be finished. Otherwise consumers would have to assume
692
            # that it still has not completely shutdown.
693
            return self.wait()
661
        except AttributeError:
694
        except AttributeError:
662
            # Try to print a relevant error message.
695
            # Try to print a relevant error message.
663
            if not self.proc:
696
            if not hasattr(self, 'proc'):
664
                print >> sys.stderr, "Unable to kill Process because call to ProcessHandler constructor failed."
697
                print >> sys.stderr, "Unable to kill Process because call to ProcessHandler constructor failed."
665
            else:
698
            else:
666
                raise
699
                raise
667
700
668
    def readWithTimeout(self, f, timeout):
701
    def readWithTimeout(self, f, timeout):
669
        """
702
        """
670
        Try to read a line of output from the file object *f*.
703
        Try to read a line of output from the file object *f*.
671
704
 Lines 690-705   falling back to not using job objects fo Link Here 
690
        for handler in self.onTimeoutHandlers:
723
        for handler in self.onTimeoutHandlers:
691
            handler()
724
            handler()
692
725
693
    def onFinish(self):
726
    def onFinish(self):
694
        """Called when a process finishes without a timeout."""
727
        """Called when a process finishes without a timeout."""
695
        for handler in self.onFinishHandlers:
728
        for handler in self.onFinishHandlers:
696
            handler()
729
            handler()
697
730
731
    def poll(self):
732
        """Check if child process has terminated
733
734
        Returns the current returncode value:
735
        - None if the process hasn't terminated yet
736
        - A negative number if the process was killed by signal N (Unix only)
737
        - '0' if the process ended without failures
738
739
        """
740
        # Ensure that we first check for the outputThread status. Otherwise
741
        # we might mark the process as finished while output is still getting
742
        # processed.
743
        if self.outThread and self.outThread.isAlive():
744
            return None
745
        elif hasattr(self.proc, "returncode"):
746
            return self.proc.returncode
747
        else:
748
            return self.proc.poll()
749
698
    def processOutput(self, timeout=None, outputTimeout=None):
750
    def processOutput(self, timeout=None, outputTimeout=None):
699
        """
751
        """
700
        Handle process output until the process terminates or times out.
752
        Handle process output until the process terminates or times out.
701
753
702
        If timeout is not None, the process will be allowed to continue for
754
        If timeout is not None, the process will be allowed to continue for
703
        that number of seconds before being killed.
755
        that number of seconds before being killed.
704
756
705
        If outputTimeout is not None, the process will be allowed to continue
757
        If outputTimeout is not None, the process will be allowed to continue
 Lines 748-766   falling back to not using job objects fo Link Here 
748
        """
800
        """
749
        Waits until all output has been read and the process is
801
        Waits until all output has been read and the process is
750
        terminated.
802
        terminated.
751
803
752
        If timeout is not None, will return after timeout seconds.
804
        If timeout is not None, will return after timeout seconds.
753
        This timeout only causes the wait function to return and
805
        This timeout only causes the wait function to return and
754
        does not kill the process.
806
        does not kill the process.
755
807
756
        Returns the process' exit code. A None value indicates the
808
        Returns the process exit code value:
757
        process hasn't terminated yet. A negative value -N indicates
809
        - None if the process hasn't terminated yet
758
        the process was killed by signal N (Unix only).
810
        - A negative number if the process was killed by signal N (Unix only)
811
        - '0' if the process ended without failures
812
759
        """
813
        """
760
        if self.outThread:
814
        if self.outThread:
761
            # Thread.join() blocks the main thread until outThread is finished
815
            # Thread.join() blocks the main thread until outThread is finished
762
            # wake up once a second in case a keyboard interrupt is sent
816
            # wake up once a second in case a keyboard interrupt is sent
763
            count = 0
817
            count = 0
764
            while self.outThread.isAlive():
818
            while self.outThread.isAlive():
765
                self.outThread.join(timeout=1)
819
                self.outThread.join(timeout=1)
766
                count += 1
820
                count += 1
 Lines 838-910   falling back to not using job objects fo Link Here 
838
    @property
892
    @property
839
    def pid(self):
893
    def pid(self):
840
        return self.proc.pid
894
        return self.proc.pid
841
895
842
896
843
### default output handlers
897
### default output handlers
844
### these should be callables that take the output line
898
### these should be callables that take the output line
845
899
846
def print_output(line):
847
    print line
848
849
class StoreOutput(object):
900
class StoreOutput(object):
850
    """accumulate stdout"""
901
    """accumulate stdout"""
851
902
852
    def __init__(self):
903
    def __init__(self):
853
        self.output = []
904
        self.output = []
854
905
855
    def __call__(self, line):
906
    def __call__(self, line):
856
        self.output.append(line)
907
        self.output.append(line)
857
908
858
class LogOutput(object):
909
class StreamOutput(object):
910
    """pass output to a stream and flush"""
911
912
    def __init__(self, stream):
913
        self.stream = stream
914
915
    def __call__(self, line):
916
        self.stream.write(line + '\n')
917
        self.stream.flush()
918
919
class LogOutput(StreamOutput):
859
    """pass output to a file"""
920
    """pass output to a file"""
860
921
861
    def __init__(self, filename):
922
    def __init__(self, filename):
862
        self.filename = filename
923
        self.file_obj = open(filename, 'a')
863
        self.file = None
924
        StreamOutput.__init__(self, self.file_obj)
864
865
    def __call__(self, line):
866
        if self.file is None:
867
            self.file = file(self.filename, 'a')
868
        self.file.write(line + '\n')
869
        self.file.flush()
870
925
871
    def __del__(self):
926
    def __del__(self):
872
        if self.file is not None:
927
        if self.file_obj is not None:
873
            self.file.close()
928
            self.file_obj.close()
929
874
930
875
### front end class with the default handlers
931
### front end class with the default handlers
876
932
877
class ProcessHandler(ProcessHandlerMixin):
933
class ProcessHandler(ProcessHandlerMixin):
878
    """
934
    """
879
    Convenience class for handling processes with default output handlers.
935
    Convenience class for handling processes with default output handlers.
880
936
881
    If no processOutputLine keyword argument is specified, write all
937
    If no processOutputLine keyword argument is specified, write all
882
    output to stdout.  Otherwise, the function specified by this argument
938
    output to stdout.  Otherwise, the function or the list of functions
883
    will be called for each line of output; the output will not be written
939
    specified by this argument will be called for each line of output;
884
    to stdout automatically.
940
    the output will not be written to stdout automatically.
885
941
886
    If storeOutput==True, the output produced by the process will be saved
942
    If storeOutput==True, the output produced by the process will be saved
887
    as self.output.
943
    as self.output.
888
944
889
    If logfile is not None, the output produced by the process will be
945
    If logfile is not None, the output produced by the process will be
890
    appended to the given file.
946
    appended to the given file.
891
    """
947
    """
892
948
893
    def __init__(self, cmd, logfile=None, storeOutput=True, **kwargs):
949
    def __init__(self, cmd, logfile=None, stream=None, storeOutput=True, **kwargs):
894
        kwargs.setdefault('processOutputLine', [])
950
        kwargs.setdefault('processOutputLine', [])
895
951
        if callable(kwargs['processOutputLine']):
896
        # Print to standard output only if no outputline provided
952
            kwargs['processOutputLine'] = [kwargs['processOutputLine']]
897
        if not kwargs['processOutputLine']:
898
            kwargs['processOutputLine'].append(print_output)
899
953
900
        if logfile:
954
        if logfile:
901
            logoutput = LogOutput(logfile)
955
            logoutput = LogOutput(logfile)
902
            kwargs['processOutputLine'].append(logoutput)
956
            kwargs['processOutputLine'].append(logoutput)
903
957
958
        if stream:
959
            streamoutput = StreamOutput(stream)
960
            kwargs['processOutputLine'].append(streamoutput)
961
962
        # Print to standard output only if no outputline provided
963
        if not kwargs['processOutputLine']:
964
            kwargs['processOutputLine'].append(StreamOutput(sys.stdout))
965
904
        self.output = None
966
        self.output = None
905
        if storeOutput:
967
        if storeOutput:
906
            storeoutput = StoreOutput()
968
            storeoutput = StoreOutput()
907
            self.output = storeoutput.output
969
            self.output = storeoutput.output
908
            kwargs['processOutputLine'].append(storeoutput)
970
            kwargs['processOutputLine'].append(storeoutput)
909
971
910
        ProcessHandlerMixin.__init__(self, cmd, **kwargs)
972
        ProcessHandlerMixin.__init__(self, cmd, **kwargs)

Return to bug 1026710