big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
1e71fe8c42
hadoop/dev-support/determine-flaky-tests-hadoop.py

246 lines
8.3 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Given a jenkins test job, this script examines all runs of the job done
# within specified period of time (number of days prior to the execution
# time of this script), and reports all failed tests.
#
# The output of this script includes a section for each run that has failed
# tests, with each failed test name listed.
#
# More importantly, at the end, it outputs a summary section to list all failed
# tests within all examined runs, and indicate how many runs a same test
# failed, and sorted all failed tests by how many runs each test failed.
#
# This way, when we see failed tests in PreCommit build, we can quickly tell
# whether a failed test is a new failure, or it failed before and how often it
# failed, so to have idea whether it may just be a flaky test.
#
# Of course, to be 100% sure about the reason of a test failure, closer look
# at the failed test for the specific run is necessary.
#
import sys
import platform
sysversion = sys.hexversion
onward30 = False
if sysversion < 0x020600F0:
sys.exit("Minimum supported python version is 2.6, the current version is " +
"Python" + platform.python_version())
if sysversion == 0x030000F0:
sys.exit("There is a known bug with Python" + platform.python_version() +
", please try a different version");
if sysversion < 0x03000000:
import urllib2
else:
onward30 = True
import urllib.request
import datetime
import json as simplejson
import logging
from optparse import OptionParser
import time
# Configuration
DEFAULT_JENKINS_URL = "https://builds.apache.org"
DEFAULT_JOB_NAME = "Hadoop-Common-trunk"
DEFAULT_NUM_PREVIOUS_DAYS = 14
DEFAULT_TOP_NUM_FAILED_TEST = -1
SECONDS_PER_DAY = 86400
# total number of runs to examine
numRunsToExamine = 0
#summary mode
summary_mode = False
#total number of errors
error_count = 0
""" Parse arguments """
def parse_args():
parser = OptionParser()
parser.add_option("-J", "--jenkins-url", type="string",
dest="jenkins_url", help="Jenkins URL",
default=DEFAULT_JENKINS_URL)
parser.add_option("-j", "--job-name", type="string",
dest="job_name", help="Job name to look at",
default=DEFAULT_JOB_NAME)
parser.add_option("-n", "--num-days", type="int",
dest="num_prev_days", help="Number of days to examine",
default=DEFAULT_NUM_PREVIOUS_DAYS)
parser.add_option("-t", "--top", type="int",
dest="num_failed_tests",
help="Summary Mode, only show top number of failed tests",
default=DEFAULT_TOP_NUM_FAILED_TEST)
(options, args) = parser.parse_args()
if args:
parser.error("unexpected arguments: " + repr(args))
return options
""" Load data from specified url """
def load_url_data(url):
if onward30:
ourl = urllib.request.urlopen(url)
codec = ourl.info().get_param('charset')
content = ourl.read().decode(codec)
data = simplejson.loads(content, strict=False)
else:
ourl = urllib2.urlopen(url)
data = simplejson.load(ourl, strict=False)
return data
""" List all builds of the target project. """
def list_builds(jenkins_url, job_name):
global summary_mode
url = "%(jenkins)s/job/%(job_name)s/api/json?tree=builds[url,result,timestamp]" % dict(
jenkins=jenkins_url,
job_name=job_name)
try:
data = load_url_data(url)
except:
if not summary_mode:
logging.error("Could not fetch: %s" % url)
error_count += 1
raise
return data['builds']
""" Find the names of any tests which failed in the given build output URL. """
def find_failing_tests(testReportApiJson, jobConsoleOutput):
global summary_mode
global error_count
ret = set()
try:
data = load_url_data(testReportApiJson)
except:
if not summary_mode:
logging.error(" Could not open testReport, check " +
jobConsoleOutput + " for why it was reported failed")
error_count += 1
return ret
for suite in data['suites']:
for cs in suite['cases']:
status = cs['status']
errDetails = cs['errorDetails']
if (status == 'REGRESSION' or status == 'FAILED' or (errDetails is not None)):
ret.add(cs['className'] + "." + cs['name'])
if len(ret) == 0 and (not summary_mode):
logging.info(" No failed tests in testReport, check " +
jobConsoleOutput + " for why it was reported failed.")
return ret
""" Iterate runs of specfied job within num_prev_days and collect results """
def find_flaky_tests(jenkins_url, job_name, num_prev_days):
global numRunsToExamine
global summary_mode
all_failing = dict()
# First list all builds
builds = list_builds(jenkins_url, job_name)
# Select only those in the last N days
min_time = int(time.time()) - SECONDS_PER_DAY * num_prev_days
builds = [b for b in builds if (int(b['timestamp']) / 1000) > min_time]
# Filter out only those that failed
failing_build_urls = [(b['url'] , b['timestamp']) for b in builds
if (b['result'] in ('UNSTABLE', 'FAILURE'))]
tnum = len(builds)
num = len(failing_build_urls)
numRunsToExamine = tnum
if not summary_mode:
logging.info(" THERE ARE " + str(num) + " builds (out of " + str(tnum)
+ ") that have failed tests in the past " + str(num_prev_days) + " days"
+ ((".", ", as listed below:\n")[num > 0]))
for failed_build_with_time in failing_build_urls:
failed_build = failed_build_with_time[0];
jobConsoleOutput = failed_build + "Console";
testReport = failed_build + "testReport";
testReportApiJson = testReport + "/api/json";
ts = float(failed_build_with_time[1]) / 1000.
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
if not summary_mode:
logging.info("===>%s" % str(testReport) + " (" + st + ")")
failing = find_failing_tests(testReportApiJson, jobConsoleOutput)
if failing:
for ftest in failing:
if not summary_mode:
logging.info(" Failed test: %s" % ftest)
all_failing[ftest] = all_failing.get(ftest,0)+1
return all_failing
def main():
global numRunsToExamine
global summary_mode
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
# set up logger to write to stdout
soh = logging.StreamHandler(sys.stdout)
soh.setLevel(logging.INFO)
logger = logging.getLogger()
logger.removeHandler(logger.handlers[0])
logger.addHandler(soh)
opts = parse_args()
logging.info("****Recently FAILED builds in url: " + opts.jenkins_url
+ "/job/" + opts.job_name + "")
if opts.num_failed_tests != -1:
summary_mode = True
all_failing = find_flaky_tests(opts.jenkins_url, opts.job_name,
opts.num_prev_days)
if len(all_failing) == 0:
raise SystemExit(0)
if summary_mode and opts.num_failed_tests < len(all_failing):
logging.info("\nAmong " + str(numRunsToExamine) +
" runs examined, top " + str(opts.num_failed_tests) +
" failed tests <#failedRuns: testName>:")
else:
logging.info("\nAmong " + str(numRunsToExamine) +
" runs examined, all failed tests <#failedRuns: testName>:")
# print summary section: all failed tests sorted by how many times they failed
line_count = 0
for tn in sorted(all_failing, key=all_failing.get, reverse=True):
logging.info(" " + str(all_failing[tn])+ ": " + tn)
if summary_mode:
line_count += 1
if line_count == opts.num_failed_tests:
break
if summary_mode and error_count > 0:
logging.info("\n" + str(error_count) + " errors found, you may "
+ "re-run in non summary mode to see error details.");
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink