big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12135. cleanup releasedocmaker

Browse Source 
(cherry picked from commit 3fee9f8d18dd60d83da674b3cfbefe666915fad8)
This commit is contained in:
Allen Wittenauer 2015-07-06 15:49:03 -07:00 committed by Andrew Wang
parent 098ba450cc
commit e8b62d11d4
1 changed files with 195 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
dev-support/releasedocmaker.py
View File

@ -19,6 +19,7 @@
from glob import glob
from optparse import OptionParser
from time import gmtime, strftime
import pprint
import os
import re
import sys
@ -99,23 +100,44 @@ def mstr(obj):
return ""
return unicode(obj)
def buildindex(master):
def buildindex(title,license):
versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
with open("index.md","w") as indexfile:
if license is True:
indexfile.write(asflicense)
for v in versions:
indexfile.write("* Apache Hadoop v%s\n" % (v))
indexfile.write("* %s v%s\n" % (title,v))
for k in ("Changes","Release Notes"):
indexfile.write(" * %s\n" %(k))
indexfile.write(" * [Combined %s](%s/%s.%s.html)\n" \
indexfile.write(" * %s (%s/%s.%s.html)\n" \
% (k,v,k.upper().replace(" ",""),v))
if not master:
indexfile.write(" * [Hadoop Common %s](%s/%s.HADOOP.%s.html)\n" \
% (k,v,k.upper().replace(" ",""),v))
for p in ("HDFS","MapReduce","YARN"):
indexfile.write(" * [%s %s](%s/%s.%s.%s.html)\n" \
% (p,k,v,k.upper().replace(" ",""),p.upper(),v))
indexfile.close()
class GetVersions:
""" yo """
def __init__(self,versions, projects):
versions = versions
projects = projects
self.newversions = []
pp = pprint.PrettyPrinter(indent=4)
at=0
end=1
count=100
versions.sort()
print "Looking for %s through %s"%(versions[0],versions[-1])
for p in projects:
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
data = json.loads(resp.read())
for d in data:
if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
print "Adding %s to the list" % d['name']
self.newversions.append(d['name'])
newlist=list(set(self.newversions))
self.newversions=newlist
def getlist(self):
pp = pprint.PrettyPrinter(indent=4)
return(self.newversions)
class Version:
"""Represents a version number"""
def __init__(self, data):
@ -261,8 +283,10 @@ def getReleaseDate(self,version):
class JiraIter:
"""An Iterator of JIRAs"""
def __init__(self, versions):
self.versions = versions
def __init__(self, version, projects):
self.version = version
self.projects = projects
v=str(version).replace("-SNAPSHOT","")
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
data = json.loads(resp.read())
@ -276,7 +300,7 @@ def __init__(self, versions):
end=1
count=100
while (at < end):
params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
data = json.loads(resp.read())
if (data.has_key('errorMessages')):
@ -286,8 +310,6 @@ def __init__(self, versions):
self.jiras.extend(data['issues'])
needaversion=False
for j in versions:
v=str(j).replace("-SNAPSHOT","")
if v not in releaseVersion:
needaversion=True
@ -351,21 +373,29 @@ def writeList(self, mylist):
self.writeKeyRaw(jira.getProject(), line)
def main():
parser = OptionParser(usage="usage: %prog --version VERSION [--version VERSION2 ...]",
parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
epilog=
"Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
" named after the highest version provided.")
parser.add_option("-i","--index", dest="index", action="store_true",
default=False, help="build an index file")
parser.add_option("-l","--license", dest="license", action="store_false",
default=True, help="Add an ASF license")
parser.add_option("-n","--lint", dest="lint", action="store_true",
help="use lint flag to exit on failures")
parser.add_option("-p", "--project", dest="projects",
action="append", type="string",
help="projects in JIRA to include in releasenotes", metavar="PROJECT")
parser.add_option("-r", "--range", dest="range", action="store_true",
default=False, help="Given versions are a range")
parser.add_option("-t", "--projecttitle", dest="title",
type="string",
help="Title to use for the project (default is Apache PROJECT)")
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
default=False, help="use current date for unreleased versions")
parser.add_option("-v", "--version", dest="versions",
action="append", type="string",
help="versions in JIRA to include in releasenotes", metavar="VERSION")
parser.add_option("-m","--master", dest="master", action="store_true",
help="only create the master, merged project files")
parser.add_option("-i","--index", dest="index", action="store_true",
help="build an index file")
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
help="use current date for unreleased versions")
parser.add_option("-n","--lint", dest="lint", action="store_true",
help="use lint flag to exit on failures")
(options, args) = parser.parse_args()
if (options.versions is None):
@ -377,52 +407,52 @@ def main():
if (len(options.versions) <= 0):
parser.error("At least one version needs to be supplied")
versions = [ Version(v) for v in options.versions ];
projects = options.projects
if (options.range is True):
versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
else:
versions = [ Version(v) for v in options.versions ]
versions.sort();
maxVersion = str(versions[-1])
if (options.title is None):
title=projects[0]
else:
title=options.title
jlist = JiraIter(versions)
version = maxVersion
for v in versions:
vstr=str(v)
jlist = JiraIter(vstr,projects)
if version in releaseVersion:
reldate=releaseVersion[version]
if vstr in releaseVersion:
reldate=releaseVersion[vstr]
elif options.usetoday:
reldate=strftime("%Y-%m-%d", gmtime())
else:
reldate="Unreleased"
if not os.path.exists(version):
os.mkdir(version)
if not os.path.exists(vstr):
os.mkdir(vstr)
if options.master:
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
[], {"ver":maxVersion, "date":reldate})
[], {"ver":v, "date":reldate, "title":title})
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
[], {"ver":maxVersion, "date":reldate})
else:
reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
"%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
"%(ver)s/CHANGES.%(key)s.%(ver)s.md",
["HADOOP","HDFS","MAPREDUCE","YARN"], {"ver":maxVersion, "date":reldate})
[], {"ver":v, "date":reldate, "title":title})
if (options.license is True):
reloutputs.writeAll(asflicense)
choutputs.writeAll(asflicense)
relhead = '# Hadoop %(key)s %(ver)s Release Notes\n\n' \
relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
chhead = '# Hadoop Changelog\n\n' \
chhead = '# %(title)s Changelog\n\n' \
'## Release %(ver)s - %(date)s\n'\
'\n'
reloutputs.writeAll(relhead)
choutputs.writeAll(chhead)
errorCount=0
warningCount=0
lintMessage=""
@ -539,7 +569,7 @@ def main():
choutputs.close()
if options.index:
buildindex(options.master)
buildindex(title,options.license)
if __name__ == "__main__":
main()