Closed Bug 1156399 Opened 7 years ago Closed 7 years ago

Switch OrangeFactor/Logparser over to the new OF Elasticsearch cluster

Categories

(Tree Management Graveyard :: OrangeFactor, defect)

defect
Not set
normal

Tracking

(Not tracked)

RESOLVED FIXED

People

(Reporter: emorley, Assigned: emorley)

References

Details

OF is not compatible with the newer versions of ES, so a separate ES cluster has been created for OF to use, so we don't hold back everyone else. At some point in the future we'll then stop using ES entirely, since we'll make OF use Treeherder's APIs instead.

Old cluster:
elasticsearch-zlb.webapp.scl3.mozilla.com

New:
of-elasticsearch-zlb.webapp.scl3.mozilla.com
[root@orangefactor1.dmz.phx1 home]# grep elasticsearch-zlb -R -s .
...
./webtools/apps/logparser/src/mozautolog/mozautolog/buildlog.py:      self.server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/logparser/src/mozautolog/mozautolog/mozautolog.py:      self.server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/logparser/bin/savelogs-gen.sh:nohup ${SAVELOGS} --es --durable --es-server=elasticsearch-zlb.webapp.scl3.mozilla.com:9200 --savedir=${OUTPUT} --outputdir=${FINISHEDLOGS} --outputlog=${OUTPUTLOG} --errorlog=${ERRORLOG} --pidfile=${PIDFILE} --pulse-cfg=${PULSECFG} > ${ENV}/savelogs.log 2>&1 &
./webtools/apps/logparser/bin/savelogs-daemon-gen.sh:start-stop-daemon --start --user ${USER} --chuid ${USER}:${USER} --pidfile ${PIDFILE} --nicelevel 10 --make-pidfile --background --exec ${SAVELOGS} -- --es --durable --es-server=elasticsearch-zlb.webapp.scl3.mozilla.com:9200 --savedir=${OUTPUT} --outputdir=${FINISHEDLOGS} --outputlog=${OUTPUTLOG} --errorlog=${ERRORLOG} --pulse-cfg=${PULSECFG} > savelogs.log 2>&1

[root@brasstacks1.dmz.scl3 home]# grep elasticsearch-zlb -R -s .

./mcote/orangefactor/lib64/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./mcote/orangefactor/lib/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./mcote/orangefactor/src/orangefactor/server/orangefactor.conf:es = elasticsearch-zlb.webapp.scl3.mozilla.com:9200
./mcote/orangefactor/src/bzcache/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200' ./mcote/orangefactor/src/bzcache/build/lib/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./webtools/apps/orangefactor/lib64/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/orangefactor/lib/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./webtools/apps/orangefactor/src/orangefactor/server/orangefactor.conf:es = elasticsearch-zlb.webapp.scl3.mozilla.com:9200

./webtools/apps/orangefactor/src/bzcache/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/orangefactor/src/bzcache/build/lib/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./webtools/apps/orangefactor/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'bugs', 'bug_info')
./webtools/apps/orangefactor/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'logs', 'builds')
./webtools/apps/orangefactor/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'logs', ['testruns', 'testfailures'])

./webtools/apps/gofaster_dashboard/src/mozautolog/mozautolog/mozautolog.py:      self.server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/gofaster_dashboard/src/mozautolog/mozautolog/buildlog.py:      self.server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./webtools/apps/gofaster_dashboard/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'bugs', 'bug_info')
./webtools/apps/gofaster_dashboard/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'logs', 'builds')
./webtools/apps/gofaster_dashboard/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'logs', ['testruns', 'testfailures'])
./webtools/apps/autolog/src/mozautolog/mozautolog/mozautolog.py:      self.server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/src/mozautolog/mozautolog/buildlog.py:      self.server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./webtools/apps/autolog/src/bzcache/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'

./webtools/apps/autolog/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'bugs', 'bug_info')
./webtools/apps/autolog/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'logs', 'builds')
./webtools/apps/autolog/src/mozautoeslib/mozautoeslib/example.py:  eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'logs', ['testruns', 'testfailures'])

./webtools/apps/autolog/src/autolog/autolog_server.conf:es_server = elasticsearch-zlb.webapp.scl3.mozilla.com:9200
./webtools/apps/autolog/src/autolog/autolog_server.conf:bz_cache_server = elasticsearch-zlb.webapp.scl3.mozilla.com:9200

./webtools/apps/autolog/src/autolog/autolog_server.py:    server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/src/autolog/autolog_server.py:    bzserver = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/test.py:es_server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
https://hg.mozilla.org/automation/orangefactor/rev/7bb599d15aa4
https://hg.mozilla.org/automation/logparser/rev/ccac09e4c00a
https://hg.mozilla.org/automation/mozautolog/rev/a6744ba20f71
https://hg.mozilla.org/automation/mozautoeslib/rev/676a64adef22
https://github.com/jonallengriffin/bzcache/commit/7be7d5a1838089c41902d9cb20c84595663a57e9
https://github.com/jonallengriffin/bzcache/commit/66a00585fae759bba613afcb0b6952fcbb9162d7

The mozautoeslib entries in comment 1 are just examples, so we can ignore those. Also autolog is EOL and gofaster_dashboard doesn't use ES/is broken anyway. As such the only ones that needed updating were:

[webtools@brasstacks1.dmz.scl3 orangefactor]$ hg pull -u
pulling from http://hg.mozilla.org/automation/orangefactor/
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

[webtools@brasstacks1.dmz.scl3 mozautoeslib]$ hg pull -u
pulling from http://hg.mozilla.org/automation/mozautoeslib
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

[webtools@brasstacks1.dmz.scl3 ~]$ cd ~/apps/orangefactor/src/bzcache/
[webtools@brasstacks1.dmz.scl3 bzcache]$ git pull
remote: Counting objects: 16, done.
remote: Compressing objects: 100% (10/10), done.
remote: Total 16 (delta 2), reused 0 (delta 0), pack-reused 6
Unpacking objects: 100% (16/16), done.
From https://github.com/jonallengriffin/bzcache
   553bbf5..66a0058  master     -> origin/master
Updating 553bbf5..66a0058
Fast-forward
 bzcache/bzcache.py |    9 ++++++---
 bzcache/config.py  |    2 +-
 setup.py           |    2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

[webtools@brasstacks1.dmz.scl3 orangefactor]$ source bin/activate
(orangefactor)[webtools@brasstacks1.dmz.scl3 orangefactor]$ cd src/bzcache/
(orangefactor)[webtools@brasstacks1.dmz.scl3 bzcache]$ python setup.py develop
running develop
running egg_info
writing requirements to bzcache.egg-info/requires.txt
writing bzcache.egg-info/PKG-INFO
writing top-level names to bzcache.egg-info/top_level.txt
writing dependency_links to bzcache.egg-info/dependency_links.txt
reading manifest file 'bzcache.egg-info/SOURCES.txt'
writing manifest file 'bzcache.egg-info/SOURCES.txt'
running build_ext
Creating /home/webtools/apps/orangefactor/lib/python2.6/site-packages/bzcache.egg-link (link to .)
bzcache 0.1.3 is already the active version in easy-install.pth

Installed /home/webtools/apps/orangefactor/src/bzcache
Processing dependencies for bzcache==0.1.3
Searching for mozautoeslib==0.1.3
Best match: mozautoeslib 0.1.3
mozautoeslib 0.1.3 is already the active version in easy-install.pth

Using /home/webtools/apps/orangefactor/src/mozautoeslib
Searching for pyes==0.16.0
Best match: pyes 0.16.0
Processing pyes-0.16.0-py2.6.egg
pyes 0.16.0 is already the active version in easy-install.pth

Using /home/webtools/apps/orangefactor/lib/python2.6/site-packages/pyes-0.16.0-py2.6.egg
Finished processing dependencies for bzcache==0.1.3

[mcote@brasstacks1.dmz.scl3 orangefactor]$ hg pull -u
pulling from http://hg.mozilla.org/automation/orangefactor
searching for changes
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 2 files
2 files updated, 0 files merged, 0 files removed, 0 files unresolved

[mcote@brasstacks1.dmz.scl3 bzcache]$ git pull
fatal: Not a git repository (or any of the parent directories): .git
-> is really old (pre git move), presumably unused. Think the files under mcote's account are just for the mailer script, which doesn't touch ES directly (it uses the OF API iirc)

[root@brasstacks1.dmz.scl3 orangefactor]# ./deploy.sh
[root@brasstacks1.dmz.scl3 orangefactor]# /etc/init.d/orangefactor stop; /etc/init.d/orangefactor start; /etc/init.d/ng inx reload
stopping orangefactor                                      [  OK  ]
starting orangefactorspawn-fcgi: child spawned successfully: PID: 7756
                                                           [  OK  ]
Reloading nginx:                                           [  OK  ]


Now for orangefactor1...

[webtools@orangefactor1.dmz.phx1 logparser]$ hg pull -u
pulling from http://hg.mozilla.org/automation/logparser
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 2 changes to 2 files
2 files updated, 0 files merged, 0 files removed, 0 files unresolved

[webtools@orangefactor1.dmz.phx1 mozautoeslib]$ hg pull -u
pulling from http://hg.mozilla.org/automation/mozautoeslib/
searching for changes
adding changesets
adding manifests
adding file changes
added 4 changesets with 4 changes to 2 files
2 files updated, 0 files merged, 0 files removed, 0 files unresolved

[webtools@orangefactor1.dmz.phx1 mozautolog]$ hg pull -u
pulling from https://hg.mozilla.org/automation/mozautolog
searching for changes
adding changesets
adding manifests
adding file changes
added 4 changesets with 6 changes to 4 files
4 files updated, 0 files merged, 0 files removed, 0 files unresolved

[webtools@orangefactor1.dmz.phx1 mozautolog]$ sed -i s/elasticsearch-zlb.webapp.scl3.mozilla.com/of-elasticsearch-zlb.w ebapp.scl3.mozilla.com/ ~/apps/logparser/bin/savelogs-gen.sh

[webtools@orangefactor1.dmz.phx1 mozautolog]$ sed -i s/elasticsearch-zlb.webapp.scl3.mozilla.com/of-elasticsearch-zlb.w ebapp.scl3.mozilla.com/ ~/apps/logparser/bin/savelogs-daemon-gen.sh

[root@orangefactor1.dmz.phx1 home]# /etc/init.d/logparser restart
stopping logparser                                         [  OK  ]
starting logparser                                         [  OK  ]
Looks like everything that matters has now been changed:

[root@orangefactor1.dmz.phx1 home]# grep -P '(?<!of-)elasticsearch-zlb' -R -I --exclude=".bash_history" .
[root@orangefactor1.dmz.phx1 home]#

Think I needed to have done a |setup.py install| not develop above, so did that now:

(orangefactor)[webtools@brasstacks1.dmz.scl3 bzcache]$ python setup.py install
running install
...
creating /home/webtools/apps/orangefactor/lib/python2.6/site-packages/bzcache-0.1.3-py2.6.egg
Extracting bzcache-0.1.3-py2.6.egg to /home/webtools/apps/orangefactor/lib/python2.6/site-packages
Removing bzcache 0.1.3 from easy-install.pth file
Adding bzcache 0.1.3 to easy-install.pth file

Installed /home/webtools/apps/orangefactor/lib/python2.6/site-packages/bzcache-0.1.3-py2.6.egg
Processing dependencies for bzcache==0.1.3
Searching for mozautoeslib==0.1.3
Best match: mozautoeslib 0.1.3
mozautoeslib 0.1.3 is already the active version in easy-install.pth

Using /home/webtools/apps/orangefactor/src/mozautoeslib
Searching for pyes==0.16.0
Best match: pyes 0.16.0
Processing pyes-0.16.0-py2.6.egg
pyes 0.16.0 is already the active version in easy-install.pth

Using /home/webtools/apps/orangefactor/lib/python2.6/site-packages/pyes-0.16.0-py2.6.egg
Finished processing dependencies for bzcache==0.1.3


[root@brasstacks1.dmz.scl3 home]# grep -P '(?<!of-)elasticsearch-zlb' -R -I --exclude=".bash_history" .
./mcote/orangefactor/lib64/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./mcote/orangefactor/lib/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./mcote/orangefactor/src/bzcache/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200' ./mcote/orangefactor/src/bzcache/build/lib/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/orangefactor/lib64/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/orangefactor/lib/python2.6/site-packages/bzcache-0.1.0-py2.6.egg/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/src/bzcache/bzcache/config.py:DEFAULT_ES_SERVER = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/src/autolog/autolog_server.conf:es_server = elasticsearch-zlb.webapp.scl3.mozilla.com:9200
./webtools/apps/autolog/src/autolog/autolog_server.conf:bz_cache_server = elasticsearch-zlb.webapp.scl3.mozilla.com:9200

./webtools/apps/autolog/src/autolog/autolog_server.py:    server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/src/autolog/autolog_server.py:    bzserver = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./webtools/apps/autolog/test.py:es_server = 'elasticsearch-zlb.webapp.scl3.mozilla.com:9200'
./jgriffin/gaia_es.py:    eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'autolog', ['testgroups'])
./jgriffin/gaia_es.py:    eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'autolog', ['testfailures'])
./rwood/queryES.py:    eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'autolog', ['testgroups'])
./rwood/queryES.py:    eslib = ESLib('elasticsearch-zlb.webapp.scl3.mozilla.com:9200', 'autolog', ['testfailures'])
Oh and to kick-start the bzcache index creation:

[webtools@brasstacks1.dmz.scl3 ~]$ . ~/apps/orangefactor/bin/activate
(orangefactor)[webtools@brasstacks1.dmz.scl3 ~]$ python ~/apps/orangefactor/src/bzcache/bzcache/bz_cache_refresh.py
No handlers could be found for logger "pyes.urllib3.connectionpool"
2015-04-20, 12:48:40 - 453969 added, status: RESOLVED, id: 453969
2015-04-20, 12:48:40 - 470420 added, status: RESOLVED, id: 470420
...
2015-04-20, 12:52:09 - 1156437 added, status: NEW, id: 1156437
2015-04-20, 12:52:09 - 1156446 added, status: NEW, id: 1156446
2015-04-20 12:52:09.224677 http://brasstacks.mozilla.com/orangefactor/api/bybug?startday=2015-02-19&endday=2015-04-20&tree=All

I've checked the logs and everything looks fine with logparser ingestion.
Status: ASSIGNED → RESOLVED
Closed: 7 years ago
Resolution: --- → FIXED
And now that bug 1156448 is on treeherder-stage, and I've submitted a bug classification, http://brasstacks.mozilla.com/orangefactor/ is loading fine (previously it was 500ing, presumably since it doesn't handle the case where the indexes don't exist, or there are zero results returned).
Product: Tree Management → Tree Management Graveyard
You need to log in before you can comment on or make changes to this bug.