Where is the _id getting mixed up with _id?

RESOLVED FIXED

Status

Testing
ActiveData
RESOLVED FIXED
2 years ago
2 years ago

People

(Reporter: ekyle, Assigned: ekyle)

Tracking

Firefox Tracking Flags

(Not tracked)

Details

(Assignee)

Description

2 years ago
What's causing this?

> MapperParsingException[Provided id [i-6379facf] does not match the content one [202.389.25]];  while loading line into jobs20151007_005138

How is the `aws_id` getting into the `_id` property?  The bulk indexer does pollute the `_id`, but I not with just anything.  Also, the `other` property is more polluted than I would have imagined.  Clean it up by removing known property values, and deliberately ignoring the large arrays of junk.



thinkpad09281 - 2015-10-26 17:50:16 - threaded queue for push to elasticsearch: jobs20151007_005138 - "threads.py:767" (worker_bee) - 
WARNING: Problem with push to elasticsearch: jobs20151007_005138 pushing 8163 items to data sink
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/thread/threads.py", line 767, in worker_bee
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/thread/threads.py", line 442, in _run
caused by
	ERROR: problem sending to ES
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/env/elasticsearch.py", line 317, in extend
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/thread/threads.py", line 755, in worker_bee
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/thread/threads.py", line 442, in _run
caused by
	ERROR: 400 MapperParsingException[failed to parse [_id]]; nested: MapperParsingException[Provided id [i-6379facf] does not match the content one [202.389.25]];  while loading line into jobs20151007_005138:
{
	"run":{
		"machine":{
			"aws_type":"m3.xlarge",
			"name":"bld-linux64-spot-080",
			"pool":"bld-linux64-spot",
			"aws_id":"i-6379facf"
		},
		"key":"Thunderbird comm-aurora linux64 l10n nightly",
		"logurl":"http://ftp.mozilla.org/pub/mozilla.org/thunderbird/nightly/2015/07/2015-07-22-00-40-09-comm-aurora-l10n/comm-aurora-linux64-l10n-nightly-el-bm77-build1-build566.txt.gz"
	},
	"tags":["nightly"],
	"other":{
		"completeMarFilename":"thunderbird-41.0a2.el.linux-x86_64.complete.mar",
		"scheduler":"Thunderbird comm-aurora linux64 l10n nightly",
		"appName":"Thunderbird",
		"aws_ami_id":"ami-414b992a",
		"stage_platform":"linux64",
		"forced_clobber":false,
		"l10n_revision":"e79481e63a45",
		"slavebuilddir":"tb-c-aurora-l64-l10n-ntly-0000",
		"previous_buildid":"20150721004013",
		"builddir":"tb-c-aurora-l64-l10n-ntly-0000",
		"en_revision":[null],
		"basedir":"/builds/slave/tb-c-aurora-l64-l10n-ntly-0000",
		"appVersion":"41.0a2",
		"purge_actual":"64.12GB",
		"purge_target":"4GB",
		"previousMarFilename":"thunderbird-41.0a2.el.linux-x86_64.complete.mar",
		"master":"http://buildbot-master77.bb.releng.use1.mozilla.com:8001/",
		"placement/availability_zone":"us-east-1a",
		"partialMarFilename":"thunderbird-41.0a2.el.linux-x86_64.partial.20150721004013-20150722004009.mar",
		"completeMarSize":"42119175",
		"repository":"",
		"completeMarHash":"3eefa6cfc9efb494a00ced27e4d9f38ec414add675e5b673ab1bf4b7033abb8df1b57d2c6cc1f9ee30d706cfe91053d1f5b4024686a6b3b3e1af3a6e5a17ced5",
		"completeMarUrl":"http://ftp.mozilla.org/pub/mozilla.org/thunderbird/nightly/2015/07/2015-07-22-00-40-09-comm-aurora-l10n/thunderbird-41.0a2.el.linux-x86_64.complete.mar",
		"hashType":"sha512",
		"previous_inipath":"previous/application.ini",
		"toolsdir":"/builds/slave/tb-c-aurora-l64-l10n-ntly-0000/tools",
		"inipath":"dist/l10n-stage/thunderbird/application.ini",
		"partialInfo":[{
			"url":"http://ftp.mozilla.org/pub/mozilla.org/thunderbird/nightly/2015/07/2015-07-22-00-40-09-comm-aurora-l10n/thunderbird-41.0a2.el.linux-x86_64.partial.20150721004013-20150722004009.mar",
			"hash":"84bb4ed366354b6d793ce557df84815aab2d0732dd8983347fd4d179669b52dfc6f6fd97c323da72d65f52601bf3991541a150c92d4c2a6d487398cc48e7e930",
			"from_buildid":"20150721004013",
			"size":"89989"
		}],
		"moz_revision":"92dd75b29646",
		"partialMarUrl":"http://ftp.mozilla.org/pub/mozilla.org/thunderbird/nightly/2015/07/2015-07-22-00-40-09-comm-aurora-l10n/thunderbird-41.0a2.el.linux-x86_64.partial.20150721004013-20150722004009.mar",
		"purged_clobber":false,
		"tree":"tbaurora",
		"project":"",
		"periodic_clobber":false,
		"partialMarHash":"84bb4ed366354b6d793ce557df84815aab2d0732dd8983347fd4d179669b52dfc6f6fd97c323da72d65f52601bf3991541a150c92d4c2a6d487398cc48e7e930",
		"partialMarSize":"89989",
		"comm_revision":"07458a671842",
		"builduid":"79118e0e16854e0da2daa2dc5e767dc1"
	},
	"build":{
		"product":"thunderbird",
		"tags":[],
		"locale":"el",
		"platform":"linux64",
		"url":"http://ftp.mozilla.org/pub/mozilla.org/thunderbird/nightly/2015/07/2015-07-22-00-40-09-comm-aurora-l10n/thunderbird-41.0a2.el.linux-x86_64.tar.bz2",
		"branch":"comm-aurora",
		"date":1437525609,
		"release":"default",
		"type":[],
		"id":"20150722004009"
	},
	"action":{
		"request_time":1437555568,
		"buildbot_status":"success",
		"start_time":1437555590,
		"job_number":566,
		"reason":"This build was triggered by the successful completion of the en-US nightly.",
		"end_time":1437556027,
		"requests":[{"timestamp":1437555568,"request_id":75508140}],
		"repack":true
	},
	"_id":"202.389.25",
	"etl":{
		"timestamp":1445860067.951267,
		"source":{
			"timestamp":1445773642.618223,
			"type":"join",
			"id":389,
			"source":{
				"url":"http://builddata.pub.build.mozilla.org/builddata/buildjson/builds-2015-07-22.js.gz",
				"timestamp":1445773397.967512,
				"type":"join",
				"id":202
			}
		},
		"file":"http://ftp.mozilla.org/pub/mozilla.org/thunderbird/nightly/2015/07/2015-07-22-00-40-09-comm-aurora-l10n/comm-aurora-linux64-l10n-nightly-el-bm77-build1-build566.txt.gz",
		"error":"Text log does not exist",
		"type":"join",
		"id":25,
		"revision":"6323d66da43a9455cd2f1d1a2b07f921639d0903"
	}


	File "/home/klahnakoski/TestLog-ETL/pyLibrary/env/elasticsearch.py", line 307, in extend
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/thread/threads.py", line 755, in worker_bee
	File "/home/klahnakoski/TestLog-ETL/pyLibrary/thread/threads.py", line 442, in _run
(Assignee)

Comment 1

2 years ago
There's the problem:

https://github.com/klahnakoski/TestLog-ETL/blob/8d0ddb9f4314ae4b331f9af44756856d78d19a48/testlog_etl/sinks/multi_day_index.py#L94

Marking as fixed, pushed to branches soon enough.
(Assignee)

Updated

2 years ago
Status: NEW → RESOLVED
Last Resolved: 2 years ago
Resolution: --- → FIXED
You need to log in before you can comment on or make changes to this bug.