package com.atlassian.plugin.osgi.container.felix;
   
   import com.atlassian.plugin.osgi.container.PackageScannerConfiguration;
   import com.atlassian.plugin.osgi.hostcomponents.HostComponentRegistration;
   import com.atlassian.plugin.osgi.util.OsgiHeaderUtil;
   import com.atlassian.plugin.util.PluginFrameworkUtils;
   import com.google.common.annotations.VisibleForTesting;
   import com.google.common.collect.ImmutableList;
   import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner;
  import org.apache.commons.io.FileUtils;
  import org.dom4j.Document;
  import org.dom4j.DocumentException;
  import org.dom4j.Element;
  import org.dom4j.io.SAXReader;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.twdata.pkgscanner.DefaultOsgiVersionConverter;
  import org.twdata.pkgscanner.ExportPackage;
  import org.twdata.pkgscanner.PackageScanner;
  
  import javax.servlet.ServletContext;
  import java.io.File;
  import java.io.IOException;
  import java.net.MalformedURLException;
  import java.net.URL;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.Deque;
  import java.util.HashMap;
  import java.util.LinkedList;
  import java.util.List;
  import java.util.Map;
  import java.util.StringTokenizer;
  import java.util.jar.Attributes;
  import java.util.jar.JarFile;
  import java.util.jar.Manifest;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import static com.atlassian.plugin.osgi.container.felix.ExportBuilderUtils.copyUnlessExist;
  import static com.atlassian.plugin.osgi.container.felix.ExportBuilderUtils.parseExportFile;
  import static com.google.common.collect.Lists.newArrayList;
  import static org.twdata.pkgscanner.PackageScanner.exclude;
  import static org.twdata.pkgscanner.PackageScanner.include;
  import static org.twdata.pkgscanner.PackageScanner.jars;
  import static org.twdata.pkgscanner.PackageScanner.packages;
  
  /**
   * Builds the OSGi package exports string. Uses a file to cache the scanned results, keyed by the application version.
   */
  class ExportsBuilder {
  
      static final String JDK8_PACKAGES_PATH = "jdk8-packages.txt";
      static final String JDK9_PACKAGES_PATH = "jdk9-packages.txt";
      static final String JDK11_PACKAGES_PATH = "jdk11-packages.txt";
  
      private static final List<String> FRAMEWORK_PACKAGES = ImmutableList.of(
              "com.atlassian.plugin.remotable",
              // webfragments and webresources moved out in PLUG-942 and PLUG-943
              "com.atlassian.plugin.cache.filecache",
              "com.atlassian.plugin.webresource",
              "com.atlassian.plugin.web"
      );
      private static final String OSGI_PACKAGES_PATH = "osgi-packages.txt";
      /**
       * A pattern for {@code jar:file:} URLs which extracts the file path. {@code jar:file:} URLs are common in
       * Spring Boot's {@code URLClassLoader}s.
       *
       * @see #maybeUnwrapJarFileUrl(URL)
       */
      private static final Pattern PATTERN_JAR_FILE_URL = Pattern.compile("jar:(file:.+\\.jar)!/");
      private static final Logger log = LoggerFactory.getLogger(ExportsBuilder.class);
  
  
      static String getLegacyScanModeProperty() {
          return "com.atlassian.plugin.export.legacy.scan.mode";
      }
  
      private static String exportStringCache;
  
      public interface CachedExportPackageLoader {
          Collection<ExportPackage> load();
      }
  
      private final CachedExportPackageLoader cachedExportPackageLoader;
  
      ExportsBuilder() {
          this(new PackageScannerExportsFileLoader("package-scanner-exports.xml"));
      }
  
      ExportsBuilder(final CachedExportPackageLoader loader) {
          this.cachedExportPackageLoader = loader;
      }
  
      /**
       * Detects URLs of the form {@code jar:file:some/path/to.jar!/} and unwraps them to
       * {@code file:some/path/to.jar}. While both denote paths to jar files on disk, the
       * {@code PackageScanner} code only accepts the latter. {@code jar:file:} URLs are
      * used extensively by Spring Boot, and this unwrapping allows the jars to be scanned.
      *
      * @param url the URL to unwrap
      * @return the unwrapped URL, if it matched the required pattern, or the provided URL
      *         unchanged if it did not match the pattern or the unwrapped URL was invalid
      */
     @VisibleForTesting
     static URL maybeUnwrapJarFileUrl(final URL url) {
         final Matcher matcher = PATTERN_JAR_FILE_URL.matcher(url.toString());
         if (matcher.matches()) {
             final String fileUrl = matcher.group(1);
             log.debug("Unwrapped Spring Boot URL: {} -> {}", url, fileUrl);
 
             try {
                 return new URL(fileUrl);
             } catch (final MalformedURLException e) {
                 log.warn("Could not create URL from apparent Spring Boot jar:file: {}->{}", url, fileUrl);
             }
         }
 
         return url;
     }
 
     @VisibleForTesting
     static boolean isPluginFrameworkPackage(String pkg) {
         return pkg.startsWith("com.atlassian.plugin.") &&
                 FRAMEWORK_PACKAGES.stream().noneMatch(
                         frameworkPackage -> pkg.equals(frameworkPackage) || pkg.startsWith(frameworkPackage + "."));
     }
 
     /**
      * Gets the framework exports taking into account host components and package scanner configuration.
      * <p>
      * Often, this information will not change without a system restart, so we determine this once and then cache the value.
      * The cache is only useful if the plugin system is thrown away and re-initialised. This is done thousands of times
      * during JIRA functional testing, and the cache was added to speed this up.
      *
      * If needed, call {@link #clearExportCache()} to clear the cache.
      *
      * @param regs                 The list of host component registrations
      * @param packageScannerConfig The configuration for the package scanning
      * @return A list of exports, in a format compatible with OSGi headers
      */
     String getExports(final List<HostComponentRegistration> regs, final PackageScannerConfiguration packageScannerConfig) {
         if (exportStringCache == null) {
             exportStringCache = determineExports(regs, packageScannerConfig);
         }
         return exportStringCache;
     }
 
     /**
      * Clears the export string cache. This results in {@link #getExports(List, PackageScannerConfiguration)}
      * having to recalculate the export string next time which can significantly slow down the start up time of plugin framework.
      *
      * @since 2.9.0
      */
     void clearExportCache() {
         exportStringCache = null;
     }
 
     /**
      * Determines framework exports taking into account host components and package scanner configuration.
      *
      * @param regs                 The list of host component registrations
      * @param packageScannerConfig The configuration for the package scanning
      * @return A list of exports, in a format compatible with OSGi headers
      */
     String determineExports(final List<HostComponentRegistration> regs, final PackageScannerConfiguration packageScannerConfig) {
         final Map<String, String> exportPackages = new HashMap<>();
 
         // The first part is osgi related packages.
         copyUnlessExist(exportPackages, parseExportFile(OSGI_PACKAGES_PATH));
 
         // The second part is JDK packages.
         copyUnlessExist(exportPackages, parseExportFile(getJdkPackagesPath()));
 
         // Third part by scanning packages available via classloader. The versions are determined by jar names.
         final Collection<ExportPackage> scannedPackages = generateExports(packageScannerConfig);
         copyUnlessExist(exportPackages, ExportBuilderUtils.toMap(scannedPackages));
 
         // Fourth part by scanning host components since all the classes referred to by them must be available to consumers.
         try {
             final Map<String, String> referredPackages = OsgiHeaderUtil.findReferredPackageVersions(
                     regs, packageScannerConfig.getPackageVersions());
             copyUnlessExist(exportPackages, referredPackages);
         } catch (final IOException ex) {
             log.error("Unable to calculate necessary exports based on host components", ex);
         }
 
         // All the packages under plugin framework namespace must be exported as the plugin framework's version.
         enforceFrameworkVersion(exportPackages);
 
         // Generate the actual export string in OSGi spec.
         final String exports = OsgiHeaderUtil.generatePackageVersionString(exportPackages);
 
         if (log.isDebugEnabled()) {
             log.debug("Exports:\n" + exports.replaceAll(",", "\r\n"));
         }
 
         return exports;
     }
 
     private void enforceFrameworkVersion(final Map<String, String> exportPackages) {
         final String frameworkVersion = PluginFrameworkUtils.getPluginFrameworkVersion();
 
         // convert the version to OSGi format.
         final DefaultOsgiVersionConverter converter = new DefaultOsgiVersionConverter();
         final String frameworkVersionOsgi = converter.getVersion(frameworkVersion);
 
         exportPackages.keySet().stream()
                 .filter(ExportsBuilder::isPluginFrameworkPackage)
                 .forEach(pkg -> exportPackages.put(pkg, frameworkVersionOsgi));
     }
 
     Collection<ExportPackage> generateExports(final PackageScannerConfiguration packageScannerConfig) {
         final String[] arrType = new String[0];
 
         final Map<String, String> pkgVersions = new HashMap<>(packageScannerConfig.getPackageVersions());
         // If the product isn't trying to override servlet, set the version to that reported by the ServletContext
         final String javaxServletPattern = "javax.servlet*";
         final ServletContext servletContext = packageScannerConfig.getServletContext();
         if ((null == pkgVersions.get(javaxServletPattern)) && (null != servletContext)) {
             final String servletVersion = servletContext.getMajorVersion() + "." + servletContext.getMinorVersion();
             pkgVersions.put(javaxServletPattern, servletVersion);
         }
 
         final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
         final PackageScanner scanner = new PackageScanner()
                 .useClassLoader(contextClassLoader)
                 .select(
                         jars(
                                 include(packageScannerConfig.getJarIncludes().toArray(arrType)),
                                 exclude(packageScannerConfig.getJarExcludes().toArray(arrType))),
                         packages(
                                 include(packageScannerConfig.getPackageIncludes().toArray(arrType)),
                                 exclude(packageScannerConfig.getPackageExcludes().toArray(arrType)))
                 )
                 .withMappings(pkgVersions);
 
         if (log.isDebugEnabled()) {
             scanner.enableDebug();
         }
 
         Collection<ExportPackage> exports = cachedExportPackageLoader.load();
         if (exports == null) {
             final boolean legacyMode = Boolean.getBoolean(getLegacyScanModeProperty());
             if (legacyMode) {
                 // Legacy mode is still in used on windows bamboo agents - see https://extranet.atlassian.com/jira/browse/BDEV-8619
                 // The issue is that the path walking code org.twdata.pkgscanner.InternalScanner#loadImplementationsInDirectory
                 // assumes that (File.isDirectory() == true) => (File.listFiles[] != null), which is not true for windows Junctions.
                 exports = scanner.scan();
             } else {
                 final URL[] urls = getClassPathUrls(contextClassLoader);
                 exports = scanner.scan(urls);
             }
         }
         log.info("Package scan completed. Found " + exports.size() + " packages to export.");
 
         if (packageScanFailed(exports) && servletContext != null) {
             log.warn("Unable to find expected packages via classloader scanning. Trying ServletContext scanning...");
             try {
                 exports = scanner.scan(servletContext.getResource("/WEB-INF/lib"), servletContext.getResource("/WEB-INF/classes"));
             } catch (final MalformedURLException e) {
                 log.warn("Unable to scan webapp for packages", e);
             }
         }
 
         if (packageScanFailed(exports)) {
             throw new IllegalStateException("Unable to find required packages via classloader or servlet context"
                     + " scanning, most likely due to an application server bug.");
         }
         return exports;
     }
 
     private URL[] getClassPathUrls(ClassLoader contextClassLoader) {
         // The stack of urls we have yet to expand. We initialize the deque with the URLs in the order the class loader
         // searches them, since the ArrayDeque implementation pops from the front.
         final Deque<URL> loaderUrls = new LinkedList<>();
 
         new FastClasspathScanner("")
                 .addClassLoader(contextClassLoader)
                 .disableRecursiveScanning()
                 .getUniqueClasspathElementURLs()
                 .forEach(loaderUrls::push);
 
         // Expand the list of loaderUrls to obtain all jar and directory entries accessible from them.
         // A best effort is made to return the results in the order they are loaded by class loading. We deal only in
         // file (jar and directory) entries, because PackageScanner only works on these anyway. For reasons
         // inexplicable, intermittently in some environments urls such as "http://felix.extensions:9/" appear in the
         // classpath.
 
         final List<URL> allUrls = new ArrayList<>();
         while (!loaderUrls.isEmpty()) {
             final URL url = maybeUnwrapJarFileUrl(loaderUrls.pop());
             try {
                 final File file = FileUtils.toFile(url);
                 if (null == file) {
                     log.warn("Cannot deep scan non file '{}'", url);
                 } else if (!file.exists()) {
                     // This is only debug worthy - jars sometimes speculatively reference optional components
                     log.debug("Cannot deep scan missing file '{}'", url);
                 } else if (file.isDirectory()) {
                     // A directory class path entry is fine, so collect it, but we can't scan any deeper into it
                     allUrls.add(url);
                 } else if (file.isFile() && file.getName().endsWith(".jar")) {
                     // It's a jar file, so collect it ...
                     allUrls.add(url);
                     // ... and look for a Class-Path to expand
                     final JarFile jar = new JarFile(file);
                     collectClassPath(loaderUrls, url, jar);
                 } else {
                     // This is reasonable, for example there's JNI stuff in the class path we can't deep scan,
                     // but a log message seems reasonable just so everything can be tracked when debugging.
                     log.debug("Skipping deep scan of non jar-file ");
                 }
             } catch (final Exception exception) {
                 // The likely reasons for hitting this, based on inspection at the time of writing, are
                 // 1. An unparseable URL in the Class-Path in a manifest, and
                 // 2. An IOException opening a JAR
                 // Neither of these are expected during normal usage, nor have happened during testing, so the current
                 // plan is to ignore them and push on. Things not definitely critically broken at this point, the likely
                 // outcome is just that some jars didn't get scanned. It may be that we need to fallback to the mode
                 // used when getLegacyScanModeProperty is set, but it's not yet clear what failure modes are extreme
                 // enough to warrant this, since both the above cases have the same expected outcome in legacy mode.
                 log.warn("Failed to deep scan '{}'", url, exception);
             }
         }
         return allUrls.toArray(new URL[0]);
     }
 
     /**
      * Collect URLs obtained by parsing the Manifest "Class-Path:" of the given jar.
      * <p>
      * We collect the results using a Deque so we can push jars found in Manifest Class-Path: entries back on the front and
      * search them before following jars.
      * <p>
      * The expansion of the Manifest Class-Path: entries adds code complexity and may impact performance, but since surefire
      * uses the Manifest Class-Path: to run tests in the correct class loaders, we need to handle it. See PLUGDEV-67 for
      * more details.
      *
      * @param loaderUrls the stack of urls to push discovered entries on to.
      * @param url        the url of the jar used to resolve relative entries.
      * @param jar        the actual jar file whose manifest is parsed.
      */
     private void collectClassPath(final Deque<URL> loaderUrls, final URL url, final JarFile jar) throws IOException {
         final Manifest manifest = jar.getManifest();
         if (null == manifest) {
             log.debug("Missing manifest prevents deep scan of '{}'", url);
             return;
         }
 
         final String classPath = manifest.getMainAttributes().getValue(Attributes.Name.CLASS_PATH);
         if (null != classPath) {
             final StringTokenizer tokenizer = new StringTokenizer(classPath);
             while (tokenizer.hasMoreTokens()) {
                 final String classPathEntry = tokenizer.nextToken();
                 try {
                     loaderUrls.push(new URL(url, classPathEntry));
                     log.debug("Deep scan found url '{}'", loaderUrls.peekFirst());
                 } catch (final MalformedURLException emu) {
                     // Classloaders silently ignore bad URLs in Class-Path, so we non-silently ignore them
                     log.warn("Cannot deep scan unparseable Class-Path entry '{}' in '{}'", url, classPath);
                 }
             }
         }
         // else no Class-Path: entry which is fine, there's no need to scan anything else
     }
 
     private String getJdkPackagesPath() {
         String versionString = System.getProperty("java.specification.version");
         if (versionString == null) {
             versionString = System.getProperty("java.version", "11");
         }
         if (versionString.startsWith("1.")) {
             versionString = versionString.substring(2);
         }
         int version = 0;
         for (char c : versionString.toCharArray()) {
             if (Character.isDigit(c)) {
                 version = 10 * version + Character.digit(c, 10);
             }
         }
 
         if (version >= 11) {
             return JDK11_PACKAGES_PATH;
         }
         if (version >= 9) {
             return JDK9_PACKAGES_PATH;
         }
         return JDK8_PACKAGES_PATH;
     }
 
     /**
      * Tests to see if a scan of packages to export was successful, using the presence of slf4j as the criteria.
      *
      * @param exports The exports found so far
      * @return True if slf4j is present, false otherwise
      */
     private static boolean packageScanFailed(final Collection<ExportPackage> exports) {
         return exports.stream().noneMatch(export -> export.getPackageName().equals("org.slf4j"));
     }
 
     static class PackageScannerExportsFileLoader implements CachedExportPackageLoader {
         private final String path;
 
         PackageScannerExportsFileLoader(final String path) {
             this.path = path;
         }
 
         @Override
         public Collection<ExportPackage> load() {
             final URL exportsUrl = getClass().getClassLoader().getResource(path);
             if (exportsUrl != null) {
                 log.debug("Precalculated exports found, loading...");
                 final List<ExportPackage> result = newArrayList();
                 try {
                     final Document doc = new SAXReader().read(exportsUrl);
                     // Our version of dom4j does not have a generic safe API
                     //noinspection unchecked
                     for (final Element export : ((List<Element>) doc.getRootElement().elements())) {
                         final String packageName = export.attributeValue("package");
                         final String version = export.attributeValue("version");
                         final String location = export.attributeValue("location");
 
                         if (packageName == null || location == null) {
                             log.warn("Invalid configuration: package({}) and location({}) are required, " +
                                             "aborting precalculated exports and reverting to normal scanning",
                                     packageName, location);
                             return Collections.emptyList();
                         }
                         result.add(new ExportPackage(packageName, version, new File(location)));
                     }
                     log.debug("Loaded {} precalculated exports", result.size());
 
                     return result;
                 } catch (final DocumentException e) {
                     log.warn("Unable to load exports from " + path + " due to malformed XML", e);
                 }
             }
             log.debug("No precalculated exports found");
             return null;
         }
     }
 }