Be more failsafe when creating documentation

we do not fail immediatly, instead we continue with the next step. Of course we emit a warning/error into
the log in case of a failure.

In case there is a failure the script returns with != 0

createDocu.groovy

@@ -4,14 +4,228 @@ import org.codehaus.groovy.control.CompilerConfiguration
 import com.sap.piper.DefaultValueCache
 import java.util.regex.Matcher
 
+//
+// Collects helper functions for rendering the docu
+//
+class TemplateHelper {
+
+  static createParametersTable(Map parameters) {
+
+    def t = ''
+    t += '| name | mandatory | default | possible values |\n'
+
+    parameters.keySet().toSorted().each {
+
+      def props = parameters.get(it)
+      t +=  "| `${it}` | ${props.required ? 'yes' : 'no'} | `${(props.defaultValue ?: 'n/a') }` | ${props.value ?: 'n/a'} |\n"
+    }
+
+    t
+  }
+
+  static createParameterDescriptionSection(Map parameters) {
+    def t =  ''
+    parameters.keySet().toSorted().each {
+      def props = parameters.get(it)
+      t += "* `${it}` - ${props.docu ?: 'n/a'}\n"
+    }
+
+    t
+  }
+}
+
+//
+// Collects generic helper functions
+//
+class Helper {
+
+  static getConfigHelper(classLoader, roots) {
+
+    def compilerConfig = new CompilerConfiguration()
+        compilerConfig.setClasspathList( roots )
+
+    new GroovyClassLoader(classLoader, compilerConfig, true)
+        .parseClass(new File('src/com/sap/piper/ConfigurationHelper.groovy'))
+        .newInstance()
+  }
+
+
+  static getPrepareDefaultValuesStep(def gse) {
+
+    def prepareDefaultValuesStep = gse.createScript('prepareDefaultValues.groovy', new Binding())
+
+    prepareDefaultValuesStep.metaClass.handlePipelineStepErrors {
+      m, c ->  c()
+    }
+    prepareDefaultValuesStep.metaClass.libraryResource {
+      f ->  new File("resources/${f}").text
+    }
+
+    prepareDefaultValuesStep.metaClass.readYaml {
+      m -> new Yaml().load(m.text)
+    }
+
+    prepareDefaultValuesStep
+  }
+
+  static getDummyScript(def prepareDefaultValuesStep, def stepName) {
+
+    def _prepareDefaultValuesStep = prepareDefaultValuesStep
+    def _stepName = stepName
+
+    return  new Script() {
+
+      def STEP_NAME = _stepName
+
+      def prepareDefaultValues() {
+        _prepareDefaultValuesStep()
+      }
+
+      def run() {
+        throw new UnsupportedOperationException()
+      }
+    }
+  }
+
+  private static normalize(Set p) {
+
+    def normalized = [] as Set
+
+    def interim = [:]
+    p.each {
+      def parts = it.split('/') as List
+      _normalize(parts, interim)
+    }
+
+    interim.each { k, v -> flatten (normalized, k, v)   }
+
+    normalized
+  }
+
+  private static void _normalize(List parts, Map interim) {
+    if( parts.size >= 1) {
+      if( ! interim[parts.head()]) interim[parts.head()] = [:]
+      _normalize(parts.tail(), interim[parts.head()])
+    }
+  }
+
+
+  private static flatten(Set flat, def key, Map interim) {
+
+    if( ! interim ) flat << (key as String)
+
+    interim.each { k, v ->
+
+      def _key = "${key}/${k}"
+
+      if( v && v.size() > 0 )
+        flatten(flat, _key, v)
+      else {
+        flat << (_key as String)
+      }
+    }
+  }
+
+  static void scanDocu(File f, Map params) {
+
+    boolean docu = false,
+            value = false,
+            scanNextLineForParamName = false
+
+    def docuLines = [],
+        valueLines = []
+
+    f.eachLine  {
+      line ->
+
+      if(scanNextLineForParamName) {
+          scanNextLineForParamName = false
+
+          Matcher m = (line =~ /.*withMandatoryProperty\(.*'(.*)'.*/)
+          if(m.size() == 1 && m[0].size() == 2) {
+            // otherwise there is a comment we do care for
+            def param = m[0][1]
+            def _docu = [], _value = []
+            docuLines.each { _docu << it  }
+            valueLines.each { _value << it}
+            params[param].docu = _docu*.trim().join(' ')
+            params[param].value = _value*.trim().join(' ')
+          }
+
+          docuLines.clear()
+          valueLines.clear()
+      }
+
+      if( line.trim()  ==~ /^\/\*\*/ ) {
+        docu = true
+      }
+
+      if(docu) {
+        def _line = line.replaceAll('^.*\\*/?', '').trim()
+
+        if(_line ==~ /@possibleValues.*/) {
+            value = true
+        }
+
+        if(_line) {
+          if(value) {
+            if(_line ==~ /@possibleValues.*/)
+              _line = (_line =~ /@possibleValues\s*?(.*)/)[0][1]
+            valueLines << _line
+          } else {
+            docuLines << _line
+          }
+        }
+      }
+
+      if(docu && line.trim() ==~ /^\*\//) {
+        docu = false
+        value = false
+        scanNextLineForParamName = true
+      }
+    }
+  }
+
+
+  static getScopedParameters(def script) {
+
+    def params = [:]
+
+    params.put('STEP_CONFIG', script.STEP_CONFIG_KEYS ?: [])
+    params.put('GENERAL_CONFIG', script.GENERAL_CONFIG_KEYS ?: [] )
+    params.put('PARAMS', script.PARAMETER_KEYS ?: [] )
+
+    return params
+  }
+
+  static getRequiredParameters(File f) {
+    def params = [] as Set
+    f.eachLine  {
+      line ->
+      if( line ==~ /.*withMandatoryProperty.*/ ) {
+        def param = (line =~ /.*withMandatoryProperty\('(.*)'/)[0][1]
+        params << param
+      }
+    }
+    return params
+  }
+
+  static getValue(Map config, def pPath) {
+    def p =config[pPath.head()]
+    if(pPath.size() == 1) return p // there is no tail
+    if(p in Map) getValue(p, pPath.tail())
+    else return p
+  }
+}
+
 roots = [
     'vars',
     'src',
     ]
 
-def stepsDir
+stepsDir = null
-def outDir
+outDir = null
-def stepDocuDir
+stepsDocuDir = null
 
 //
 // assign parameters
@@ -57,27 +271,15 @@ if( !stepsDir.exists() ) {
 // sanity checks
 //
 
-def blacklist = [
+blacklist = [
                   'toolValidate',
-                  'transportRequestCreate',
-                  'cloudFoundryDeploy',
-                  'artifactSetVersion',
-                  'testsPublishResults',
-                  'transportRequestUploadFile',
-                  'snykExecute', // docu does not exist
-                  'batsExecuteTests',
-                  'transportRequestRelease',
                   'setupCommonPipelineEnvironment',
-                  'dockerExecute',
                   'durationMeasure',
                   'prepareDefaultValues',
                   'pipelineStashFilesAfterBuild',
                   'pipelineStashFiles',
                   'handlePipelineStepErrors',
-                  'newmanExecute',
                   'commonPipelineEnvironment',
-                  'pipelineStashFilesBeforeBuild',
-                  'seleniumExecuteTests',
                   'pipelineExecute',
                 ]
 
@@ -90,19 +292,47 @@ stepsDir.traverse(type: FileType.FILES, maxDepth: 0)
 
 def gse = new GroovyScriptEngine( [ stepsDir.getName()  ] as String[] , getClass().getClassLoader() )
 
-def prepareDefaultValuesStep = getPrepareDefaultValuesStep(gse)
+def prepareDefaultValuesStep = Helper.getPrepareDefaultValuesStep(gse)
+
+boolean exceptionCaught = false
 
 for (step in steps) {
+  try {
+    handleStep(step, prepareDefaultValuesStep, gse)
+  } catch(Exception e) {
+    exceptionCaught = true
+    System.err << "${e.getClass().getName()} caught while handling step '${step}'."
+  }
+}
+if(exceptionCaught) {
+  System.err << "[ERROR] Exception caught during generating documentation. Check earlier log for details.\n"
+  System.exit(1)
+}
+
+
+void handleStep(step, prepareDefaultValuesStep, gse) {
+  File theStep = new File(stepsDir, "${step}.groovy")
+  File theStepDocuInput = new File(stepsDocuDir, "${step}.md")
+  File theGeneratedStepDocu = new File(outDir, "${step}.md")
+
+  if(!theStepDocuInput.exists()) {
+    System.err << "[WARNING] step docu input file for step '${step}' is missing.\n"
+    return
+  }
 
   if(blacklist.contains(step)) {
-    // better: simply copy over the input file to the output file directory ...
+    // in case a file is blacklisted from docu generation we simply copy over
+    // the docu input file in order to have always a complete set of step docu files.
+    // we do not touch the content of the file since we assume it has been carefully handcrafted.
+    theGeneratedStepDocu << theStepDocuInput.text
+
     System.err << "[INFO] Step '${step}' is blacklisted. No docu will be created for that step.\n"
-    continue
+    return
   }
 
   System.err << "[INFO] Handling step '${step}'.\n"
 
-  def defaultConfig = getConfigHelper().loadStepDefaults(getDummyScript(prepareDefaultValuesStep, step)).use()
+  def defaultConfig = Helper.getConfigHelper(getClass().getClassLoader(), roots).loadStepDefaults(Helper.getDummyScript(prepareDefaultValuesStep, step)).use()
 
   def params = [] as Set
 
@@ -113,25 +343,25 @@ for (step in steps) {
   def scopedParameters
 
   try {
-    scopedParameters = getScopedParameters(gse.createScript( "${step}.groovy", new Binding() ))
+    scopedParameters = Helper.getScopedParameters(gse.createScript( "${step}.groovy", new Binding() ))
     scopedParameters.each { k, v -> params.addAll(v) }
   } catch(Exception e) {
-    System.err << "Step '${step}' violates naming convention for scoped parameters: ${e}."
+    System.err << "[ERROR] Step '${step}' violates naming convention for scoped parameters: ${e}.\n"
-    System.exit(1)
+    throw e
   }
-  def requiredParameters = getRequiredParameters(new File(stepsDir, "${step}.groovy"))
+  def requiredParameters = Helper.getRequiredParameters(theStep)
 
   params.addAll(requiredParameters)
 
 
   def parameters = [:]
 
-  normalize(params).toSorted().each {
+  Helper.normalize(params).toSorted().each {
 
     it ->
 
     def parameterProperties = [
-                                defaultValue: getValue(defaultConfig, it.split('/')),
+                                defaultValue: Helper.getValue(defaultConfig, it.split('/')),
                                 required: requiredParameters.contains((it as String))
                               ]
 
@@ -145,215 +375,13 @@ for (step in steps) {
     }
   }
 
-  scanDocu(new File(stepsDir, "${step}.groovy"), parameters)
+  Helper.scanDocu(theStep, parameters)
 
-  def text = new File(stepsDocuDir, "${step}.md").text
+  def text = theStepDocuInput.text
-  text = text.replace('__PARAMETER_TABLE__', createParametersTable(parameters))
+  text = text.replace('__PARAMETER_TABLE__', TemplateHelper.createParametersTable(parameters))
-  text = text.replace('__PARAMETER_DESCRIPTION__', createParameterDescriptionSection(parameters))
+  text = text.replace('__PARAMETER_DESCRIPTION__', TemplateHelper.createParameterDescriptionSection(parameters))
 
-  new File(outDir, "${step}.md").withWriter { w -> w.write text }
+  theGeneratedStepDocu.withWriter { w -> w.write text }
 }
 
 System.err << "[INFO] done.\n"
-
-def getConfigHelper() {
-
-    def compilerConfig = new CompilerConfiguration()
-        compilerConfig.setClasspathList( roots )
-
-    new GroovyClassLoader(getClass().getClassLoader(), compilerConfig, true)
-        .parseClass(new File('src/com/sap/piper/ConfigurationHelper.groovy'))
-        .newInstance()
-}
-
-def getPrepareDefaultValuesStep(def gse) {
-
-  def prepareDefaultValuesStep = gse.createScript('prepareDefaultValues.groovy', new Binding())
-
-  prepareDefaultValuesStep.metaClass.handlePipelineStepErrors {
-    m, c ->  c()
-  }
-  prepareDefaultValuesStep.metaClass.libraryResource {
-    f ->  new File("resources/${f}").text
-  }
-
-  prepareDefaultValuesStep.metaClass.readYaml {
-    m -> new Yaml().load(m.text)
-  }
-
-  prepareDefaultValuesStep
-}
-
-def getDummyScript(def prepareDefaultValuesStep, def stepName) {
-
-  def _prepareDefaultValuesStep = prepareDefaultValuesStep
-  def _stepName = stepName
-
-  return  new Script() {
-
-    def STEP_NAME = _stepName
-
-    def prepareDefaultValues() {
-      _prepareDefaultValuesStep()
-    }
-
-    def run() {
-      throw new UnsupportedOperationException()
-    }
-  }
-}
-
-def createParametersTable(Map parameters) {
-
-  def t = ''
-  t += '| name | mandatory | default | possible values |\n'
-
-  parameters.keySet().toSorted().each {
-
-    def props = parameters.get(it)
-    t +=  "| `${it}` | ${props.required ? 'yes' : 'no'} | `${(props.defaultValue ?: 'n/a') }` | ${props.value ?: 'n/a'} |\n"
-  }
-
-  t
-}
-
-def createParameterDescriptionSection(Map parameters) {
-  def t =  ''
-  parameters.keySet().toSorted().each {
-    def props = parameters.get(it)
-    t += "* `${it}` - ${props.docu ?: 'n/a'}\n"
-  }
-
-  t
-}
-
-
-def normalize(Set p) {
-
-  def normalized = [] as Set
-
-  def interim = [:]
-  p.each {
-    def parts = it.split('/') as List
-    _normalize(parts, interim)
-  }
-
-  interim.each { k, v -> flatten (normalized, k, v)   }
-
-  normalized
-}
-
-def flatten(Set flat, def key, Map interim) {
-
-  if( ! interim ) flat << (key as String)
-
-  interim.each { k, v ->
-
-     def _key = "${key}/${k}"
-
-     if( v && v.size() > 0 )
-       flatten(flat, _key, v)
-     else {
-       flat << (_key as String)
-     }
-  }
-}
-
-def _normalize(List parts, Map interim) {
-    if( parts.size >= 1) {
-      if( ! interim[parts.head()]) interim[parts.head()] = [:]
-      _normalize(parts.tail(), interim[parts.head()])
-    }
-}
-
-
-void scanDocu(File f, Map params) {
-
-    boolean docu = false,
-            value = false,
-            scanNextLineForParamName = false
-
-    def docuLines = [],
-        valueLines = []
-
-    f.eachLine  {
-      line ->
-
-      if(scanNextLineForParamName) {
-          scanNextLineForParamName = false
-
-          Matcher m = (line =~ /.*withMandatoryProperty\(.*'(.*)'.*/)
-          if(m.size() == 1 && m[0].size() == 2) {
-            // otherwise there is a comment we do care for
-            def param = m[0][1]
-            def _docu = [], _value = []
-            docuLines.each { _docu << it  }
-            valueLines.each { _value << it}
-            params[param].docu = _docu*.trim().join(' ')
-            params[param].value = _value*.trim().join(' ')
-          }
-
-          docuLines.clear()
-          valueLines.clear()
-      }
-
-      if( line.trim()  ==~ /^\/\*\*/ ) {
-        docu = true
-    }
-
-    if(docu) {
-        def _line = line.replaceAll('^.*\\*/?', '').trim()
-
-        if(_line ==~ /@possibleValues.*/) {
-            value = true
-        }
-
-        if(_line) {
-          if(value) {
-            if(_line ==~ /@possibleValues.*/)
-              _line = (_line =~ /@possibleValues\s*?(.*)/)[0][1]
-            valueLines << _line
-          } else {
-            docuLines << _line
-          }
-        }
-    }
-
-    if(docu && line.trim() ==~ /^\*\//) {
-        docu = false
-        value = false
-        scanNextLineForParamName = true
-    }
-  }
-}
-
-def getScopedParameters(def script) {
-
-  def params = [:]
-
-  params.put('STEP_CONFIG', script.STEP_CONFIG_KEYS ?: [])
-  params.put('GENERAL_CONFIG', script.GENERAL_CONFIG_KEYS ?: [] )
-  params.put('PARAMS', script.PARAMETER_KEYS ?: [] )
-
-  return params
-}
-
-def getRequiredParameters(File f) {
-  def params = [] as Set
-  f.eachLine  {
-    line ->
-    if( line ==~ /.*withMandatoryProperty.*/ ) {
-        def param = (line =~ /.*withMandatoryProperty\('(.*)'/)[0][1]
-        params << param
-    }
-  }
-  return params
-}
-
-def getValue(Map config, def pPath) {
-    def p =config[pPath.head()]
-    if(pPath.size() == 1) return p // there is no tail
-    if(p in Map) getValue(p, pPath.tail())
-    else return p
-}
-