Add the buildcontext builder from gesellix/docker-client

engine/build.gradle.kts

@@ -77,6 +77,7 @@ dependencies {
   testImplementation("com.squareup.okhttp3:mockwebserver")
 
   implementation("org.apache.commons:commons-compress:1.20")
+  testImplementation("org.apache.commons:commons-lang3:3.12.0")
 
   implementation("de.gesellix:docker-filesocket:2021-06-06T17-29-35")
   testImplementation("de.gesellix:testutil:2021-04-21T23-29-14")

engine/src/main/java/de/gesellix/docker/builder/BuildContextBuilder.java

@@ -0,0 +1,86 @@
+package de.gesellix.docker.builder;
+
+import de.gesellix.util.IOUtils;
+import okio.Okio;
+import okio.Source;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
+
+public class BuildContextBuilder {
+
+  private static final Logger log = LoggerFactory.getLogger(BuildContextBuilder.class);
+
+  public static void archiveTarFilesRecursively(File base, File targetFile) throws IOException {
+    List<File> filenames = new DockerignoreFileFilter(base, new ArrayList<>(Collections.singletonList(targetFile.getAbsolutePath()))).collectFiles(base);
+    log.debug("found {} files in buildContext.", filenames.size());
+    archiveTarFiles(base, filenames.stream().map(File::getAbsolutePath).collect(Collectors.toList()), targetFile);
+  }
+
+  public static void archiveTarFilesRecursively(File base, OutputStream target) throws IOException {
+    List<File> filenames = new DockerignoreFileFilter(base, new ArrayList<>()).collectFiles(base);
+    log.debug("found {} files in buildContext.", filenames.size());
+    archiveTarFiles(base, filenames.stream().map(File::getAbsolutePath).collect(Collectors.toList()), target);
+  }
+
+  public static void archiveTarFiles(File base, List<String> filenames, File targetFile) throws IOException {
+    archiveTarFiles(base, filenames, new FileOutputStream(targetFile));
+  }
+
+  public static void archiveTarFiles(File base, List<String> filenames, OutputStream target) throws IOException {
+    try (TarArchiveOutputStream tos = new TarArchiveOutputStream(new GZIPOutputStream(target))) {
+      tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+      for (String filename : filenames) {
+        String relativeFileName = relativize(base, new File(filename));
+        log.debug("adding {} as {}", filename, relativeFileName);
+        addAsTarEntry(new File(filename), relativeFileName, tos);
+      }
+    }
+  }
+
+  public static void addAsTarEntry(File file, String relativeFileName, TarArchiveOutputStream tos) throws IOException {
+    TarArchiveEntry tarEntry = new TarArchiveEntry(file);
+    tarEntry.setName(relativeFileName);
+
+    if (!file.isDirectory()) {
+      if (Files.isExecutable(file.toPath())) {
+        tarEntry.setMode(tarEntry.getMode() | 0755);
+      }
+    }
+
+    tos.putArchiveEntry(tarEntry);
+
+    if (!file.isDirectory()) {
+      copyFile(file, tos);
+    }
+
+    tos.closeArchiveEntry();
+  }
+
+  public static String relativize(File base, File absolute) {
+    return base.toPath().relativize(absolute.toPath()).toString();
+  }
+
+  public static long copyFile(File input, OutputStream output) throws IOException {
+    Source source = null;
+    try {
+      source = Okio.source(input);
+      return IOUtils.copy(source, Okio.sink(output));
+    }
+    finally {
+      IOUtils.closeQuietly(source);
+    }
+  }
+}

engine/src/main/java/de/gesellix/docker/builder/DockerignoreFileFilter.java

@@ -0,0 +1,101 @@
+package de.gesellix.docker.builder;
+
+import de.gesellix.util.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class DockerignoreFileFilter {
+
+  private static final Logger log = LoggerFactory.getLogger(DockerignoreFileFilter.class);
+
+  private final GlobsMatcher globsMatcher;
+
+  public DockerignoreFileFilter(File base) {
+    this(base, Collections.emptyList());
+  }
+
+  public DockerignoreFileFilter(File base, List<String> additionalExcludes) {
+    List<String> dockerignore = getDockerignorePatterns(base);
+    dockerignore.add(".dockerignore");
+    dockerignore.addAll(additionalExcludes);
+    try {
+      dockerignore = relativize(dockerignore, base);
+    }
+    catch (IllegalArgumentException e) {
+      log.error(String.format("base: %1$s, dockerignore: %2$s", base.getAbsolutePath(), dockerignore), e);
+      throw e;
+    }
+
+    log.debug("base: {}", base.getAbsolutePath());
+    log.debug("dockerignore: {}", dockerignore);
+    globsMatcher = new GlobsMatcher(base, dockerignore);
+  }
+
+  public List<String> getDockerignorePatterns(final File base) {
+    List<String> result = new ArrayList<>();
+    File[] files = base.listFiles();
+    if (files == null || files.length == 0) {
+      return result;
+    }
+    Optional<File> dockerignoreFile = Arrays.stream(files).filter((file -> {
+      String relativeFileName = relativize(base, file);
+      return ".dockerignore".equals(relativeFileName);
+    })).findFirst();
+    if (!dockerignoreFile.isPresent()) {
+      return result;
+    }
+    try {
+      Collections.addAll(result, IOUtils.toString(new FileInputStream(dockerignoreFile.get())).split("[\r\n]+"));
+      return result;
+    }
+    catch (IOException e) {
+      log.error("Couldn't read {}", dockerignoreFile.get());
+      throw new RuntimeException(e);
+    }
+  }
+
+  public List<String> relativize(Collection<String> dockerignores, final File base) {
+    return dockerignores.stream()
+        .map((String dockerignore) -> new File(dockerignore).isAbsolute() ? relativize(base, new File(dockerignore)) : dockerignore)
+        .collect(Collectors.toList());
+  }
+
+  public String relativize(File base, File absolute) {
+    Path basePath = base.getAbsoluteFile().toPath();
+    Path otherPath = absolute.getAbsoluteFile().toPath();
+    if (!basePath.getRoot().equals(otherPath.getRoot())) {
+      // Can occur on Windows, when
+      // - java temp directory is under C:/
+      // - project directory is under D:/
+      return otherPath.toString();
+    }
+
+    return basePath.relativize(otherPath).toString();
+  }
+
+  public List<File> collectFiles(File base) throws IOException {
+    final List<File> files = new ArrayList<>();
+    Files.walk(base.toPath())
+        .filter(p -> Files.isRegularFile(p) && !getGlobsMatcher().matches(p.toFile()))
+        .forEach(p -> files.add(p.toFile()));
+    log.debug("filtered list of files: {}", files);
+    return files;
+  }
+
+  public GlobsMatcher getGlobsMatcher() {
+    return globsMatcher;
+  }
+}

engine/src/main/java/de/gesellix/docker/builder/GlobsMatcher.java

@@ -0,0 +1,117 @@
+package de.gesellix.docker.builder;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class GlobsMatcher {
+
+  private static final Logger log = LoggerFactory.getLogger(GlobsMatcher.class);
+
+  private final File base;
+  private final List<String> globs;
+  private List<Matcher> matchers;
+
+  public GlobsMatcher(File base, List<String> globs) {
+    this.base = base;
+    this.globs = globs;
+  }
+
+  public void initMatchers() {
+    if (this.matchers == null) {
+      final FileSystem fileSystem = FileSystems.getDefault();
+      this.matchers = new ArrayList<>();
+      globs.stream()
+          .flatMap(glob -> {
+            if (glob.endsWith("/")) {
+              return Stream.of(
+                  new Matcher(fileSystem, glob.replaceAll("/$", "")),
+                  new Matcher(fileSystem, glob.replaceAll("/$", "/**")));
+            }
+            else {
+              return Stream.of(new Matcher(fileSystem, glob));
+            }
+          })
+          .collect(Collectors.toCollection(ArrayDeque::new))
+          .descendingIterator() // reverse the stream
+          .forEachRemaining(matchers::add);
+      if (log.isDebugEnabled()) {
+        matchers.forEach((m) -> log.debug("pattern: " + m.getPattern()));
+      }
+    }
+  }
+
+  public boolean matches(File path) {
+    initMatchers();
+
+    final Path relativePath = base.getAbsoluteFile().toPath().relativize(path.getAbsoluteFile().toPath());
+    Optional<Matcher> matcher = matchers.stream().filter((m) -> m.matches(relativePath)).findFirst();
+    if (!matcher.isPresent() && relativePath.getParent() != null) {
+      matcher = matchers.stream().filter((m) -> m.matches(relativePath.getParent())).findFirst();
+    }
+
+    return matcher.isPresent() && !matcher.get().getNegate();
+  }
+
+  public List<Matcher> getMatchers() {
+    return matchers;
+  }
+
+  public static class Matcher implements PathMatcher {
+
+    private final String pattern;
+    private final PathMatcher matcher;
+    private final boolean negate;
+
+    public Matcher(FileSystem fileSystem, String pattern) {
+      // According to https://docs.docker.com/engine/reference/builder/#dockerignore-file
+      // and https://golang.org/pkg/path/filepath/#Clean we clean paths
+      // by removing trailing slashes and also by replacing slashes with the path separator.
+      String replacement = File.separatorChar == '\\' ? "\\\\" : File.separatorChar + "";
+      this.pattern = String.join(replacement, pattern.replaceAll("/", replacement).split(replacement));
+
+      String negation = "!";
+      this.negate = pattern.startsWith(negation);
+      if (this.negate) {
+        String invertedPattern = this.pattern.substring(negation.length());
+        this.matcher = createGlob(fileSystem, invertedPattern);
+      }
+      else {
+        this.matcher = createGlob(fileSystem, this.pattern);
+      }
+    }
+
+    public static PathMatcher createGlob(FileSystem fileSystem, final String glob) {
+      return fileSystem.getPathMatcher("glob:" + glob);
+    }
+
+    @Override
+    public boolean matches(Path path) {
+      return matcher.matches(path);
+    }
+
+    public String getPattern() {
+      return pattern;
+    }
+
+    public boolean getNegate() {
+      return negate;
+    }
+
+    @Override
+    public String toString() {
+      return "matching " + (negate ? "!" : "") + pattern;
+    }
+  }
+}