HADOOP-7561. Make test-patch only run tests for changed modules.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1161291 13f79535-47bb-0310-9956-ffa450edef68

dev-support/smart-apply-patch.sh

@@ -59,15 +59,15 @@ PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
 if [[ -d hadoop-common ]]; then
   echo Looks like this is being run at project root
 
-# if all of the lines start with hadoop-common/, hadoop-hdfs/, or mapreduce/, this is
+# if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/, this is
 # relative to the hadoop root instead of the subproject root, so we need
 # to chop off another layer
-elif [[ "$PREFIX_DIRS" =~ ^(hadoop-hdfs|hadoop-common|mapreduce)$ ]]; then
+elif [[ "$PREFIX_DIRS" =~ ^(hadoop-alfredo|hadoop-annotations|hadoop-common|hadoop-hdfs|hadoop-mapreduce)$ ]]; then
 
   echo Looks like this is relative to project root. Increasing PLEVEL
   PLEVEL=$[$PLEVEL + 1]
 
-elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common\|hadoop-hdfs\|mapreduce' ; then
+elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-alfredo\|hadoop-annotations\|hadoop-common\|hadoop-hdfs\|hadoop-mapreduce' ; then
   echo Looks like this is a cross-subproject patch. Try applying from the project root
   exit 1
 fi

dev-support/test-patch.sh

@@ -446,7 +446,7 @@ checkReleaseAuditWarnings () {
   echo ""
   echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
   $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
-  find . -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
+  find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
 
   ### Compare trunk and patch release audit warning numbers
   if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
@@ -549,7 +549,7 @@ checkFindbugsWarnings () {
     $FINDBUGS_HOME/bin/convertXmlToText -html \
       $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
       $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
-    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/target/newPatchFindbugsWarnings${module_suffix}.html
+    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
 $JIRA_COMMENT_FOOTER"
   done
 
@@ -567,42 +567,83 @@ $JIRA_COMMENT_FOOTER"
 }
 
 ###############################################################################
-### Run the test-core target
+### Run the tests
-runCoreTests () {
+runTests () {
   echo ""
   echo ""
   echo "======================================================================"
   echo "======================================================================"
-  echo "    Running core tests."
+  echo "    Running tests."
   echo "======================================================================"
   echo "======================================================================"
   echo ""
   echo ""
   
-  ### Kill any rogue build processes from the last attempt
+  failed_tests=""
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+  modules=$(findModules)
-  PreTestTarget=""
+  for module in $modules;
-  if [[ $defect == MAPREDUCE-* ]] ; then
+  do
-     PreTestTarget="create-c++-configure"
+    pushd $module
-  fi
+      echo "    Running tests in $module"
+      ### Kill any rogue build processes from the last attempt
+      $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
 
-  echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
+      echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
-  $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
+      $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
-  if [[ $? != 0 ]] ; then
+      if [[ $? != 0 ]] ; then
-    ### Find and format names of failed tests
+        ### Find and format names of failed tests
-    failed_tests=`grep -l -E "<failure|<error" $WORKSPACE/trunk/target/hadoop-common/surefire-reports/*.xml | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
+        module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
+        failed_tests="${failed_tests}
+${module_failed_tests}"
+      fi
+    popd
+  done
+  echo $failed_tests
+  
+  if [[ -n "$failed_tests" ]] ; then
+  
     JIRA_COMMENT="$JIRA_COMMENT
 
-    -1 core tests.  The patch failed these core unit tests:
+    -1 core tests.  The patch failed these unit tests:
 $failed_tests"
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
 
-    +1 core tests.  The patch passed core unit tests."
+    +1 core tests.  The patch passed unit tests in $modules."
   return 0
 }
 
+###############################################################################
+### Find the modules changed by the patch
+
+findModules () {
+  # Come up with a list of changed files into $TMP
+  TMP=/tmp/tmp.paths.$$
+  $GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq > $TMP
+
+  # if all of the lines start with a/ or b/, then this is a git patch that
+  # was generated without --no-prefix
+  if ! $GREP -qv '^a/\|^b/' $TMP ; then
+    sed -i -e 's,^[ab]/,,' $TMP
+  fi
+
+  PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
+
+  # if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/, this is
+  # relative to the hadoop root instead of the subproject root
+  if [[ "$PREFIX_DIRS" =~ ^(hadoop-alfredo|hadoop-annotations|hadoop-common|hadoop-hdfs|hadoop-mapreduce)$ ]]; then
+    echo $PREFIX_DIRS
+    return 0
+  elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-alfredo\|hadoop-annotations\|hadoop-common\|hadoop-hdfs\|hadoop-mapreduce' ; then
+    echo $PREFIX_DIRS
+    return 0
+  fi
+  
+  # No modules found. Running from current directory.
+  echo .
+}
+
 ###############################################################################
 ### Run the test-contrib target
 runContribTests () {
@@ -658,6 +699,7 @@ checkInjectSystemFaults () {
   #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
   #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
   echo "NOP"
+  return 0
   if [[ $? != 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
@@ -779,7 +821,7 @@ checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))
 ### Do not call these when run by a developer 
 if [[ $JENKINS == "true" ]] ; then
-  runCoreTests
+  runTests
   (( RESULT = RESULT + $? ))
   runContribTests
   (( RESULT = RESULT + $? ))

hadoop-common/CHANGES.txt

@@ -340,6 +340,8 @@ Trunk (unreleased changes)
     HADOOP-7496. Break Maven TAR & bintar profiles into just LAYOUT & TAR proper.
     (Alejandro Abdelnur via tomwhite)
 
+    HADOOP-7561. Make test-patch only run tests for changed modules. (tomwhite)
+
   OPTIMIZATIONS
   
     HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole