HADOOP-12135. cleanup releasedocmaker
(cherry picked from commit 3fee9f8d18dd60d83da674b3cfbefe666915fad8)
This commit is contained in:
Allen Wittenauer
Andrew Wang
parent
098ba450cc
commit
e8b62d11d4
@ -19,6 +19,7 @@
|
||||
from glob import glob
|
||||
from optparse import OptionParser
|
||||
from time import gmtime, strftime
|
||||
import pprint
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@ -99,23 +100,44 @@ def mstr(obj):
|
||||
return ""
|
||||
return unicode(obj)
|
||||
|
||||
def buildindex(master):
|
||||
def buildindex(title,license):
|
||||
versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
|
||||
with open("index.md","w") as indexfile:
|
||||
if license is True:
|
||||
indexfile.write(asflicense)
|
||||
for v in versions:
|
||||
indexfile.write("* Apache Hadoop v%s\n" % (v))
|
||||
indexfile.write("* %s v%s\n" % (title,v))
|
||||
for k in ("Changes","Release Notes"):
|
||||
indexfile.write(" * %s\n" %(k))
|
||||
indexfile.write(" * [Combined %s](%s/%s.%s.html)\n" \
|
||||
indexfile.write(" * %s (%s/%s.%s.html)\n" \
|
||||
% (k,v,k.upper().replace(" ",""),v))
|
||||
if not master:
|
||||
indexfile.write(" * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \
|
||||
% (k,v,k.upper().replace(" ",""),v))
|
||||
for p in ("HDFS","MapReduce","YARN"):
|
||||
indexfile.write(" * [%s %s](%s/%s.%s.%s.html)\n" \
|
||||
% (p,k,v,k.upper().replace(" ",""),p.upper(),v))
|
||||
indexfile.close()
|
||||
|
||||
class GetVersions:
|
||||
""" yo """
|
||||
def __init__(self,versions, projects):
|
||||
versions = versions
|
||||
projects = projects
|
||||
self.newversions = []
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
at=0
|
||||
end=1
|
||||
count=100
|
||||
versions.sort()
|
||||
print "Looking for %s through %s"%(versions[0],versions[-1])
|
||||
for p in projects:
|
||||
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
|
||||
data = json.loads(resp.read())
|
||||
for d in data:
|
||||
if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
|
||||
print "Adding %s to the list" % d['name']
|
||||
self.newversions.append(d['name'])
|
||||
newlist=list(set(self.newversions))
|
||||
self.newversions=newlist
|
||||
|
||||
def getlist(self):
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
return(self.newversions)
|
||||
|
||||
class Version:
|
||||
"""Represents a version number"""
|
||||
def __init__(self, data):
|
||||
@ -261,8 +283,10 @@ def getReleaseDate(self,version):
|
||||
class JiraIter:
|
||||
"""An Iterator of JIRAs"""
|
||||
|
||||
def __init__(self, versions):
|
||||
self.versions = versions
|
||||
def __init__(self, version, projects):
|
||||
self.version = version
|
||||
self.projects = projects
|
||||
v=str(version).replace("-SNAPSHOT","")
|
||||
|
||||
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
|
||||
data = json.loads(resp.read())
|
||||
@ -276,7 +300,7 @@ def __init__(self, versions):
|
||||
end=1
|
||||
count=100
|
||||
while (at < end):
|
||||
params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
|
||||
params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
|
||||
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
|
||||
data = json.loads(resp.read())
|
||||
if (data.has_key('errorMessages')):
|
||||
@ -286,10 +310,8 @@ def __init__(self, versions):
|
||||
self.jiras.extend(data['issues'])
|
||||
|
||||
needaversion=False
|
||||
for j in versions:
|
||||
v=str(j).replace("-SNAPSHOT","")
|
||||
if v not in releaseVersion:
|
||||
needaversion=True
|
||||
if v not in releaseVersion:
|
||||
needaversion=True
|
||||
|
||||
if needaversion is True:
|
||||
for i in range(len(data['issues'])):
|
||||
@ -351,21 +373,29 @@ def writeList(self, mylist):
|
||||
self.writeKeyRaw(jira.getProject(), line)
|
||||
|
||||
def main():
|
||||
parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]",
|
||||
parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
|
||||
epilog=
|
||||
"Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
|
||||
" named after the highest version provided.")
|
||||
parser.add_option("-i","--index", dest="index", action="store_true",
|
||||
default=False, help="build an index file")
|
||||
parser.add_option("-l","--license", dest="license", action="store_false",
|
||||
default=True, help="Add an ASF license")
|
||||
parser.add_option("-n","--lint", dest="lint", action="store_true",
|
||||
help="use lint flag to exit on failures")
|
||||
parser.add_option("-p", "--project", dest="projects",
|
||||
action="append", type="string",
|
||||
help="projects in JIRA to include in releasenotes", metavar="PROJECT")
|
||||
parser.add_option("-r", "--range", dest="range", action="store_true",
|
||||
default=False, help="Given versions are a range")
|
||||
parser.add_option("-t", "--projecttitle", dest="title",
|
||||
type="string",
|
||||
help="Title to use for the project (default is Apache PROJECT)")
|
||||
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
|
||||
default=False, help="use current date for unreleased versions")
|
||||
parser.add_option("-v", "--version", dest="versions",
|
||||
action="append", type="string",
|
||||
help="versions in JIRA to include in releasenotes", metavar="VERSION")
|
||||
parser.add_option("-m","--master", dest="master", action="store_true",
|
||||
help="only create the master, merged project files")
|
||||
parser.add_option("-i","--index", dest="index", action="store_true",
|
||||
help="build an index file")
|
||||
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
|
||||
help="use current date for unreleased versions")
|
||||
parser.add_option("-n","--lint", dest="lint", action="store_true",
|
||||
help="use lint flag to exit on failures")
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if (options.versions is None):
|
||||
@ -377,169 +407,169 @@ def main():
|
||||
if (len(options.versions) <= 0):
|
||||
parser.error("At least one version needs to be supplied")
|
||||
|
||||
versions = [ Version(v) for v in options.versions ];
|
||||
projects = options.projects
|
||||
|
||||
if (options.range is True):
|
||||
versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
|
||||
else:
|
||||
versions = [ Version(v) for v in options.versions ]
|
||||
versions.sort();
|
||||
|
||||
maxVersion = str(versions[-1])
|
||||
|
||||
jlist = JiraIter(versions)
|
||||
version = maxVersion
|
||||
|
||||
if version in releaseVersion:
|
||||
reldate=releaseVersion[version]
|
||||
elif options.usetoday:
|
||||
reldate=strftime("%Y-%m-%d", gmtime())
|
||||
if (options.title is None):
|
||||
title=projects[0]
|
||||
else:
|
||||
reldate="Unreleased"
|
||||
title=options.title
|
||||
|
||||
if not os.path.exists(version):
|
||||
os.mkdir(version)
|
||||
for v in versions:
|
||||
vstr=str(v)
|
||||
jlist = JiraIter(vstr,projects)
|
||||
|
||||
if options.master:
|
||||
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
|
||||
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
|
||||
[], {"ver":maxVersion, "date":reldate})
|
||||
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
|
||||
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
|
||||
[], {"ver":maxVersion, "date":reldate})
|
||||
else:
|
||||
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
|
||||
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
|
||||
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
|
||||
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
|
||||
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
|
||||
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
|
||||
|
||||
reloutputs.writeAll(asflicense)
|
||||
choutputs.writeAll(asflicense)
|
||||
|
||||
relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
|
||||
'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
|
||||
|
||||
chhead = '# Hadoop Changelog\n\n' \
|
||||
'## Release %(ver)s - %(date)s\n'\
|
||||
'\n'
|
||||
|
||||
reloutputs.writeAll(relhead)
|
||||
choutputs.writeAll(chhead)
|
||||
|
||||
errorCount=0
|
||||
warningCount=0
|
||||
lintMessage=""
|
||||
incompatlist=[]
|
||||
buglist=[]
|
||||
improvementlist=[]
|
||||
newfeaturelist=[]
|
||||
subtasklist=[]
|
||||
tasklist=[]
|
||||
testlist=[]
|
||||
otherlist=[]
|
||||
|
||||
for jira in sorted(jlist):
|
||||
if jira.getIncompatibleChange():
|
||||
incompatlist.append(jira)
|
||||
if (len(jira.getReleaseNote())==0):
|
||||
warningCount+=1
|
||||
|
||||
if jira.checkVersionString():
|
||||
warningCount+=1
|
||||
|
||||
if jira.checkMissingComponent() or jira.checkMissingAssignee():
|
||||
errorCount+=1
|
||||
elif jira.getType() == "Bug":
|
||||
buglist.append(jira)
|
||||
elif jira.getType() == "Improvement":
|
||||
improvementlist.append(jira)
|
||||
elif jira.getType() == "New Feature":
|
||||
newfeaturelist.append(jira)
|
||||
elif jira.getType() == "Sub-task":
|
||||
subtasklist.append(jira)
|
||||
elif jira.getType() == "Task":
|
||||
tasklist.append(jira)
|
||||
elif jira.getType() == "Test":
|
||||
testlist.append(jira)
|
||||
if vstr in releaseVersion:
|
||||
reldate=releaseVersion[vstr]
|
||||
elif options.usetoday:
|
||||
reldate=strftime("%Y-%m-%d", gmtime())
|
||||
else:
|
||||
otherlist.append(jira)
|
||||
reldate="Unreleased"
|
||||
|
||||
line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
|
||||
% (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
|
||||
notableclean(jira.getSummary()))
|
||||
if not os.path.exists(vstr):
|
||||
os.mkdir(vstr)
|
||||
|
||||
if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
|
||||
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
|
||||
lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
|
||||
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
|
||||
[], {"ver":v, "date":reldate, "title":title})
|
||||
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
|
||||
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
|
||||
[], {"ver":v, "date":reldate, "title":title})
|
||||
|
||||
if jira.checkVersionString():
|
||||
lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
|
||||
if (options.license is True):
|
||||
reloutputs.writeAll(asflicense)
|
||||
choutputs.writeAll(asflicense)
|
||||
|
||||
if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
|
||||
errorMessage=[]
|
||||
jira.checkMissingComponent() and errorMessage.append("component")
|
||||
jira.checkMissingAssignee() and errorMessage.append("assignee")
|
||||
lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
|
||||
relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
|
||||
'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
|
||||
chhead = '# %(title)s Changelog\n\n' \
|
||||
'## Release %(ver)s - %(date)s\n'\
|
||||
'\n'
|
||||
|
||||
if (len(jira.getReleaseNote())>0):
|
||||
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
reloutputs.writeAll(relhead)
|
||||
choutputs.writeAll(chhead)
|
||||
errorCount=0
|
||||
warningCount=0
|
||||
lintMessage=""
|
||||
incompatlist=[]
|
||||
buglist=[]
|
||||
improvementlist=[]
|
||||
newfeaturelist=[]
|
||||
subtasklist=[]
|
||||
tasklist=[]
|
||||
testlist=[]
|
||||
otherlist=[]
|
||||
|
||||
if (options.lint is True):
|
||||
print lintMessage
|
||||
print "======================================="
|
||||
print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
|
||||
for jira in sorted(jlist):
|
||||
if jira.getIncompatibleChange():
|
||||
incompatlist.append(jira)
|
||||
if (len(jira.getReleaseNote())==0):
|
||||
warningCount+=1
|
||||
|
||||
if (errorCount>0):
|
||||
cleanOutputDir(version)
|
||||
sys.exit(1)
|
||||
if jira.checkVersionString():
|
||||
warningCount+=1
|
||||
|
||||
reloutputs.writeAll("\n\n")
|
||||
reloutputs.close()
|
||||
if jira.checkMissingComponent() or jira.checkMissingAssignee():
|
||||
errorCount+=1
|
||||
elif jira.getType() == "Bug":
|
||||
buglist.append(jira)
|
||||
elif jira.getType() == "Improvement":
|
||||
improvementlist.append(jira)
|
||||
elif jira.getType() == "New Feature":
|
||||
newfeaturelist.append(jira)
|
||||
elif jira.getType() == "Sub-task":
|
||||
subtasklist.append(jira)
|
||||
elif jira.getType() == "Task":
|
||||
tasklist.append(jira)
|
||||
elif jira.getType() == "Test":
|
||||
testlist.append(jira)
|
||||
else:
|
||||
otherlist.append(jira)
|
||||
|
||||
choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(incompatlist)
|
||||
line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
|
||||
% (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
|
||||
notableclean(jira.getSummary()))
|
||||
|
||||
choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(newfeaturelist)
|
||||
if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
|
||||
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
|
||||
lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
|
||||
choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(improvementlist)
|
||||
if jira.checkVersionString():
|
||||
lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
|
||||
|
||||
choutputs.writeAll("\n\n### BUG FIXES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(buglist)
|
||||
if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
|
||||
errorMessage=[]
|
||||
jira.checkMissingComponent() and errorMessage.append("component")
|
||||
jira.checkMissingAssignee() and errorMessage.append("assignee")
|
||||
lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
|
||||
|
||||
choutputs.writeAll("\n\n### TESTS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(testlist)
|
||||
if (len(jira.getReleaseNote())>0):
|
||||
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
|
||||
reloutputs.writeKeyRaw(jira.getProject(), line)
|
||||
|
||||
choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(subtasklist)
|
||||
if (options.lint is True):
|
||||
print lintMessage
|
||||
print "======================================="
|
||||
print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
|
||||
|
||||
choutputs.writeAll("\n\n### OTHER:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(otherlist)
|
||||
choutputs.writeList(tasklist)
|
||||
if (errorCount>0):
|
||||
cleanOutputDir(version)
|
||||
sys.exit(1)
|
||||
|
||||
choutputs.writeAll("\n\n")
|
||||
choutputs.close()
|
||||
reloutputs.writeAll("\n\n")
|
||||
reloutputs.close()
|
||||
|
||||
choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(incompatlist)
|
||||
|
||||
choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(newfeaturelist)
|
||||
|
||||
choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(improvementlist)
|
||||
|
||||
choutputs.writeAll("\n\n### BUG FIXES:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(buglist)
|
||||
|
||||
choutputs.writeAll("\n\n### TESTS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(testlist)
|
||||
|
||||
choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(subtasklist)
|
||||
|
||||
choutputs.writeAll("\n\n### OTHER:\n\n")
|
||||
choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
|
||||
choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
|
||||
choutputs.writeList(otherlist)
|
||||
choutputs.writeList(tasklist)
|
||||
|
||||
choutputs.writeAll("\n\n")
|
||||
choutputs.close()
|
||||
|
||||
if options.index:
|
||||
buildindex(options.master)
|
||||
buildindex(title,options.license)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user