770960 - find places in code where we're depending on a non-retrying call to hg to get source code, and fix them!

Status: RESOLVED INCOMPLETE

(Release Engineering :: General, defect, P3)

Description

[Chris AtLee [:catlee]]

jhopkins came up with this list:

'hg clone' operations that do not use hgtool.py.

= BUILDBOTCUSTOM =

./bin/l10n-stage/create-stage:28:    rv = subprocess.call(['hg', 'clone', r], cwd = base)

./bin/l10n-stage/create-stage:62:    rv = subprocess.call(['hg', 'clone', leaf,
./bin/l10n-stage/create-stage-63-                          os.path.join('..', 'workdir', leaf)],
./bin/l10n-stage/create-stage-64-                         cwd=base)

@ MozillaBuildFactory (addInitialSteps)
./process/factory.py-476-        self.addStep(MercurialCloneCommand(
./process/factory.py-477-         name='clone_buildtools',
./process/factory.py:478:         command=['hg', 'clone', self.buildToolsRepo, 'tools'],
./process/factory.py-479-         description=['clone', 'build tools'],
./process/factory.py-480-         workdir='.',
./process/factory.py-481-         retry=False
./process/factory.py-482-        ))

@ MercurialBuildFactory (addMultiLocaleRepoSteps)
/process/factory.py-1041-            self.addStep(MercurialCloneCommand(
./process/factory.py:1042:                name='hg_clone_%s' % name,
./process/factory.py:1043:                command=['hg', 'clone', self.getRepository(repo), name],
./process/factory.py-1044-                description=['checking', 'out', name],
./process/factory.py-1045-                descriptionDone=['checkout', name],
./process/factory.py-1046-                haltOnFailure=True,
./process/factory.py-1047-                workdir='.',
./process/factory.py-1048-            ))

@ BaseRepackFactory (getMozconfig)
./process/factory.py-3265-            self.addStep(MercurialCloneCommand(
./process/factory.py-3266-             name='checkout_configs',
./process/factory.py:3267:             command=['hg', 'clone', self.configRepo, 'configs'],
./process/factory.py-3268-             description=['checkout', 'configs'],
./process/factory.py-3269-             workdir='build/'+self.origSrcDir,
./process/factory.py-3270-             haltOnFailure=True
./process/factory.py-3271-            ))

@ BaseRepackFactory (compareLocalesSetup)
/process/factory.py-3458-        self.addStep(MercurialCloneCommand(
./process/factory.py-3459-         name='clone_compare_locales',
./process/factory.py:3460:         command=['hg', 'clone', compareLocalesRepo, 'compare-locales'],
./process/factory.py-3461-         description=['checkout', 'compare-locales'],
./process/factory.py-3462-         workdir=self.baseWorkDir,
./process/factory.py-3463-         haltOnFailure=True
./process/factory.py-3464-        ))

@ CCReleaseRepackFactory (getSources)
./process/factory.py-4056-    def getSources(self):
./process/factory.py-4057-        self.addStep(MercurialCloneCommand(
./process/factory.py-4058-         name='get_enUS_src',
./process/factory.py-4059-         command=['sh', '-c',
./process/factory.py:4060:          WithProperties('if [ -d '+self.origSrcDir+'/.hg ]; then ' +
./process/factory.py:4061:                         'hg -R '+self.origSrcDir+' pull && '+
./process/factory.py:4062:                         'hg -R '+self.origSrcDir+' up -C ;'+
./process/factory.py-4063-                         'else ' +
./process/factory.py:4064:                         'hg clone ' +
./process/factory.py:4065:                         'http://'+self.hgHost+'/'+self.repoPath+' ' +
./process/factory.py-4066-                         self.origSrcDir+' ; ' +
./process/factory.py-4067-                         'fi ' +
./process/factory.py:4068:                         '&& hg -R '+self.origSrcDir+' update -C -r %(en_revision)s')],
./process/factory.py-4069-         descriptionDone="en-US source",
./process/factory.py-4070-         workdir=self.baseWorkDir,
./process/factory.py-4071-         haltOnFailure=True,
./process/factory.py-4072-         timeout=30*60 # 30 minutes
./process/factory.py-4073-        ))

@ CCReleaseRepackFactory (getSources)
./process/factory.py-4074-        self.addStep(MercurialCloneCommand(
./process/factory.py-4075-         name='get_locale_src',
./process/factory.py-4076-         command=['sh', '-c',
./process/factory.py:4077:          WithProperties('if [ -d %(locale)s/.hg ]; then ' +
./process/factory.py:4078:                         'hg -R %(locale)s pull -r default ; ' +
./process/factory.py-4079-                         'else ' +
./process/factory.py:4080:                         'hg clone ' +
./process/factory.py:4081:                         'http://'+self.hgHost+'/'+self.l10nRepoPath+ 
./process/factory.py-4082-                           '/%(locale)s/ ; ' +
./process/factory.py-4083-                         'fi ' +
./process/factory.py:4084:                         '&& hg -R %(locale)s update -C -r %(l10n_revision)s')],
./process/factory.py-4085-         descriptionDone="locale source",
./process/factory.py-4086-         timeout=10*60, # 10 minutes
./process/factory.py-4087-         haltOnFailure=True,
./process/factory.py-4088-         workdir='%s/%s' % (self.baseWorkDir, self.l10nRepoPath)
./process/factory.py-4089-        ))

@ StagingRepositorySetupFactory (__init__)
./process/factory.py-4304-        for repoPath in sorted(repositories.keys()):
./process/factory.py-4305-            repo = self.getRepository(repoPath)
./process/factory.py-4306-            repoName = self.getRepoName(repoPath)
./process/factory.py-4307-            timeout = 60*60
./process/factory.py-4308-            command = ['python',
./process/factory.py-4309-                       WithProperties('%(toolsdir)s/buildfarm/utils/retry.py'),
./process/factory.py-4310-                       '--timeout', timeout,
./process/factory.py-4311-                       'ssh', '-l', username, '-oIdentityFile=%s' % sshKey,
./process/factory.py:4312:                       self.hgHost, 'clone', repoName, repoPath]
./process/factory.py-4313-
./process/factory.py-4314-            self.addStep(ShellCommand(
./process/factory.py-4315-             name='recreate_repo',
./process/factory.py-4316-             command=command,
./process/factory.py-4317-             description=['recreate', repoName],
./process/factory.py-4318-             timeout=timeout
./process/factory.py-4319-            ))

@ ReleaseTaggingFactory (__init__)
./process/factory.py-4465-            self.addStep(MercurialCloneCommand(
./process/factory.py:4466:             name='hg_clone',
./process/factory.py:4467:             command=['hg', 'clone', repo, repoName],
./process/factory.py-4468-             workdir='.',
./process/factory.py-4469-             description=['clone %s' % repoName],
./process/factory.py-4470-             haltOnFailure=True,
./process/factory.py-4471-             timeout=30*60 # 30 minutes
./process/factory.py-4472-            ))

@ SingleSourceFactory (addConfigSteps)
./process/factory.py-4724-        self.addStep(MercurialCloneCommand(
./process/factory.py:4725:                     name='hg_clone_configs',
./process/factory.py:4726:                     command=['hg', 'clone', configRepo, 'configs'],
./process/factory.py-4727-                     description=['checking', 'out', 'configs'],
./process/factory.py-4728-                     descriptionDone=['checkout', 'configs'],
./process/factory.py-4729-                     haltOnFailure=True,
./process/factory.py-4730-                     workdir='.'
./process/factory.py-4731-        ))

@ MultiSourceFactory (__init__)
./process/factory.py-4801-            self.addStep(MercurialCloneCommand(
./process/factory.py:4802:             name='hg_clone',
./process/factory.py:4803:             command=['hg', 'clone', repository, location],
./process/factory.py-4804-             workdir='.',
./process/factory.py-4805-             description=['clone %s' % location],
./process/factory.py-4806-             haltOnFailure=True,
./process/factory.py-4807-             timeout=30*60 # 30 minutes
./process/factory.py-4808-            ))

@ CCSourceFactory (__init__)
./process/factory.py-4901-        self.addStep(MercurialCloneCommand(
./process/factory.py:4902:         command=['hg', 'clone', self.repository, self.branchName],
./process/factory.py-4903-         workdir='.',
./process/factory.py-4904-         description=['clone %s' % self.branchName],
./process/factory.py-4905-         haltOnFailure=True,
./process/factory.py-4906-         timeout=30*60 # 30 minutes
./process/factory.py-4907-        ))

@ UnittestBuildFactory (__init__)
./process/factory.py-5744-        self.addStep(MercurialCloneCommand(
./process/factory.py-5745-         name='buildbot_configs',
./process/factory.py:5746:         command=['hg', 'clone', self.config_repo_url, 'mozconfigs'],
./process/factory.py-5747-         workdir='.'
./process/factory.py-5748-        ))

@ CCUnittestBuildFactory (__init__)
./process/factory.py-6113-        self.addStep(MercurialCloneCommand(
./process/factory.py-6114-         name='buildbot_configs',
./process/factory.py:6115:         command=['hg', 'clone', self.config_repo_url, 'mozconfigs'],
./process/factory.py-6116-         workdir='.'
./process/factory.py-6117-        ))

@ ScriptFactory (__init__)
./process/factory.py-8392-        self.addStep(ShellCommand(
./process/factory.py-8393-            name="clone_scripts",
./process/factory.py:8394:            command=[hg_bin, 'clone', scriptRepo, 'scripts'],
./process/factory.py-8395-            workdir=".",
./process/factory.py-8396-            haltOnFailure=True))

def simpleHG(host, repo, workdir, target=None):
./process/mobiletestfactory.py-64-    steps.append(ShellCommand(
./process/mobiletestfactory.py:65:        command=['hg', '-v', 'clone', '%s/%s' % (host, repo), target],
./process/mobiletestfactory.py-66-        workdir=workdir,
./process/mobiletestfactory.py-67-        description=[target, 'clone'],
./process/mobiletestfactory.py-68-        name='clone_%s' % target,
./process/mobiletestfactory.py-69-        haltOnFailure=True,
./process/mobiletestfactory.py-70-    ))

== TOOLS ==

./buildfarm/mobile/setup_foopy.sh:15:  hg clone http://hg.mozilla.org/users/tglek_mozilla.com/fennecmark [e-mail]
./buildfarm/mobile/setup_foopy.sh:16:  hg clone http://hg.mozilla.org/build/pageloader [e-mail]
./buildfarm/mobile/setup_foopy.sh:17:  hg clone http://hg.mozilla.org/build/talos talos-repo
./buildfarm/mobile/setup_foopy.sh-18-  ln -s /builds/talos-data/talos-repo/talos .
./buildfarm/mobile/setup_foopy.sh-19-fi
./buildfarm/mobile/setup_foopy.sh-20-if [ ! -d /builds/tools ]; then
./buildfarm/mobile/setup_foopy.sh-21-  cd /builds
./buildfarm/mobile/setup_foopy.sh:22:  hg clone http://hg.mozilla.org/build/tools tools

./lib/python/util/hg.py-110-def clone(repo, dest, branch=None, revision=None, update_dest=True,
…
./lib/python/util/hg.py:175:    cmd = ['hg', 'clone']

./scripts/blocklist/sync-hg-blocklist.sh-129-   echo ${HG} clone $HGREPO $REPODIR
./scripts/blocklist/sync-hg-blocklist.sh-130-        ${HG} clone $HGREPO $REPODIR

./scripts/bundle/hg-bundle.sh:15:hg clone "http://hg.mozilla.org/$REPO_PATH" clone || exit 1

./scripts/preproduction/repo_setup.py:43:        clone_repo(hgHost, hgUserName, hgSshKey, repo)

Comment 1

[Chris AtLee [:catlee]]

Lots of these references aren't actually used any more:
https://github.com/catlee/buildbotcustom/compare/mozilla:master...catlee:cleanup

Still to figure out:
* getMozconfig()
* initial tools clone for MozillabuildFactory.addInitialSteps
* StagingRepositorySetupFactory - not sure if we care
* Script factory tools clone