1 module dbuild.src;
2 
3 import std.digest.md : MD5;
4 
5 /// Source code interface
6 interface Source
7 {
8     /// Feeds the digest in a way that makes a unique build identifier.
9     void feedBuildId(ref MD5 digest);
10     /// Obtain the source code.
11     /// Implementation for local directories will simply return the local source dir,
12     /// without consideration of the workDir.
13     /// Implementation of source fetching (archive or scm) will download and
14     /// extract (or clone) the source tree into workDir.
15     /// Params:
16     ///     workDir = work directory that should preferably be used to download
17     ///               and extract the source code.
18     /// Returns: the path to the src root directory.
19     string obtain(in string workDir);
20 }
21 
22 /// Returns a Source pointing to a local existing directory
23 Source localSource(in string dir)
24 {
25     return new LocalSource(dir);
26 }
27 
28 /// Returns a Source pointing to a local directory within $DUB_PACKAGE
29 Source dubPkgSource(in string subdir)
30 {
31     import std.path : buildPath;
32     import std.process : environment;
33 
34     return new LocalSource(buildPath(environment["DUB_PACKAGE"], subdir));
35 }
36 
37 /// Returns a Source that fetches and extract an archive into the work directory
38 /// md5 can be specified with an md5 checksum to check integrity of the downloaded
39 /// archive
40 Source archiveFetchSource(in string url, in string md5=null)
41 {
42     import std.exception : enforce;
43 
44     enforce(isSupportedArchiveExt(url), url~" is not a supported archive type");
45     return new ArchiveFetchSource(url, md5);
46 }
47 
48 /// Returns a Source that will clone a git repo and checkout a specified commit.
49 /// commitRef can be a branch, tag, or commit hash. Anything `git checkout` will understand.
50 Source gitSource(in string url, in string commitRef)
51 {
52     return new GitSource(url, commitRef);
53 }
54 
55 private class LocalSource : Source
56 {
57     string dir;
58     this (in string dir)
59     {
60         import std.exception : enforce;
61         import std.file : exists, isDir;
62 
63         enforce(exists(dir) && isDir(dir), dir~": no such directory!");
64         this.dir = dir;
65     }
66 
67     override void feedBuildId(ref MD5 digest)
68     {
69         import dbuild.util : feedDigestData;
70 
71         feedDigestData(digest, dir);
72     }
73 
74     string obtain(in string)
75     {
76         return this.dir;
77     }
78 }
79 
80 private string srcLockPath(in string workDir)
81 {
82     import std.path : buildPath;
83 
84     return buildPath(workDir, ".srcLock");
85 }
86 
87 private class ArchiveFetchSource : Source
88 {
89     string url;
90     string md5;
91     this (in string url, in string md5)
92     {
93         this.url = url;
94         this.md5 = md5;
95     }
96 
97     override void feedBuildId(ref MD5 digest)
98     {
99         import dbuild.util : feedDigestData;
100 
101         feedDigestData(digest, url);
102         feedDigestData(digest, md5);
103     }
104 
105     override string obtain(in string workDir)
106     {
107         import dbuild.util : lockFile;
108         import std.file : exists, isDir;
109         import std.path : buildPath;
110         import std.uri : decode;
111 
112         const decoded = decode(url);
113         const fn = urlLastComp(decoded);
114         const archive = buildPath(workDir, fn);
115 
116         const ldn = likelySrcDirName(archive);
117         if (exists(ldn) && isDir(ldn)) {
118             return ldn;
119         }
120 
121         auto lock = lockFile(srcLockPath(workDir));
122         downloadArchive(archive);
123         return extractArchive(archive, workDir);
124     }
125 
126     private void downloadArchive(in string archive)
127     {
128         import dbuild.util : checkMd5;
129         import std.exception : enforce;
130         import std.file : exists;
131         import std.net.curl : download;
132         import std.stdio : writefln;
133 
134         if (!exists(archive) || !(md5.length && checkMd5(archive, md5))) {
135             if (exists(archive)) {
136                 import std.file : remove;
137                 remove(archive);
138             }
139             writefln("downloading %s", url);
140             download(url, archive);
141 
142             enforce(!md5.length || checkMd5(archive, md5), "wrong md5 sum for "~archive);
143         }
144     }
145 
146     private string extractArchive(in string archive, in string workDir)
147     {
148         import std.file : remove;
149 
150         final switch(archiveFormat(archive)) {
151         case ArchiveFormat.targz:
152             const tarF = extractTarGz(archive);
153             const res = extractTar(tarF, workDir);
154             remove(tarF);
155             return res;
156         case ArchiveFormat.tar:
157             return extractTar(archive, workDir);
158         case ArchiveFormat.zip:
159             return extractZip(archive, workDir);
160         }
161     }
162 
163     /// extract .tar.gz to .tar, returns the .tar file path
164     private string extractTarGz(const string archive)
165     in { assert(archive[$-7 .. $] == ".tar.gz"); }
166     out(res) { assert(res[$-4 .. $] == ".tar"); }
167     body {
168         import std.algorithm : map;
169         import std.exception : enforce;
170         import std.file : exists, remove;
171         import std.stdio : File, writefln;
172         import std.zlib : UnCompress;
173 
174         writefln("extracting %s", archive);
175 
176         const tarFn = archive[0 .. $-3];
177         enforce(!exists(tarFn), tarFn~" already exists");
178 
179         auto inF = File(archive, "rb");
180         auto outF = File(tarFn, "wb");
181 
182         UnCompress decmp = new UnCompress;
183         foreach (chunk; inF.byChunk(4096).map!(x => decmp.uncompress(x)))
184         {
185             outF.rawWrite(chunk);
186         }
187 
188         return tarFn;
189     }
190 
191     private string extractTar(in string archive, in string workDir)
192     {
193         import dbuild.tar : extractTo, isSingleRootDir;
194         import std.exception : enforce;
195         import std.file : exists, isDir;
196         import std.path : buildPath;
197         import std.stdio : writefln;
198 
199         writefln("extracting %s", archive);
200 
201         string extractDir;
202         string srcDir;
203         string rootDir;
204         if (isSingleRootDir(archive, rootDir)) {
205             extractDir = workDir;
206             srcDir = buildPath(workDir, rootDir);
207         }
208         else {
209             extractDir = buildPath(workDir, "src");
210             srcDir = extractDir;
211         }
212 
213         // trusting src dir content?
214         if (!exists(srcDir)) {
215             extractTo(archive, extractDir);
216         }
217 
218         enforce(isDir(srcDir));
219         return srcDir;
220     }
221 
222     private string extractZip(in string archive, in string workDir)
223     {
224         import std.digest.crc : crc32Of;
225         import std.exception : enforce;
226         import std.file : exists, isDir, mkdirRecurse, read, write;
227         import std.path : buildNormalizedPath, buildPath, dirName, pathSplitter;
228         import std.stdio : writeln, writefln;
229         import std.zip : ZipArchive;
230 
231         writefln("extracting %s", archive);
232         auto zip = new ZipArchive(read(archive));
233         string extractDir;
234         string srcDir;
235         string rootDir;
236         bool singleRoot = true;
237 
238         foreach(n, m; zip.directory) {
239             const dir = pathSplitter(n).front;
240             if (rootDir && dir != rootDir) {
241                 singleRoot = false;
242                 break;
243             }
244             if (!rootDir) rootDir = dir;
245         }
246         if (singleRoot) {
247             extractDir = workDir;
248             srcDir = buildPath(workDir, rootDir);
249         }
250         else {
251             extractDir = buildPath(workDir, "src");
252             srcDir = extractDir;
253         }
254 
255         foreach(n, m; zip.directory) {
256             const file = buildNormalizedPath(extractDir, n);
257             if ((exists(file) && isDir(file)) || m.expandedSize == 0) continue;
258             mkdirRecurse(dirName(file));
259             zip.expand(m);
260             enforce(m.expandedData.length == cast(size_t)m.expandedSize, "zip data does not have expected size");
261             const crc32 = crc32Of(m.expandedData);
262             const crc32_ = *(cast(const(uint)*)&crc32[0]);
263             enforce(crc32_ == m.crc32, "CRC32 zip check failed");
264             write(file, m.expandedData);
265         }
266 
267         return srcDir;
268     }
269 }
270 
271 private class GitSource : Source
272 {
273     string url;
274     string commitRef;
275 
276     this (in string url, in string commitRef)
277     {
278         import dbuild.util : searchExecutable;
279         import std.exception : enforce;
280 
281         enforce(searchExecutable("git"), "could not find git in PATH!");
282         this.url = url;
283         this.commitRef = commitRef;
284     }
285 
286     override void feedBuildId(ref MD5 digest)
287     {
288         import dbuild.util : feedDigestData;
289 
290         feedDigestData(digest, url);
291         feedDigestData(digest, commitRef);
292     }
293     override string obtain(in string workDir)
294     {
295         import dbuild.util : runCommand;
296         import std.algorithm : endsWith;
297         import std.exception : enforce;
298         import std.file : exists, isDir;
299         import std.path : buildPath;
300         import std.process : pipeProcess, Redirect;
301         import std.uri : decode;
302 
303         enforce(commitRef.length, "must specify commitRef for git checkout");
304 
305         const decoded = decode(url);
306         auto dirName = urlLastComp(decoded);
307         if (dirName.endsWith(".git")) {
308             dirName = dirName[0 .. $-4];
309         }
310 
311         const srcDir = buildPath(workDir, dirName);
312 
313         if (!exists(srcDir)) {
314             runCommand(["git", "clone", url, dirName], workDir, false);
315         }
316 
317         enforce(isDir(srcDir));
318 
319         runCommand(["git", "checkout", commitRef], srcDir, false);
320 
321         return srcDir;
322     }
323 }
324 
325 
326 private enum ArchiveFormat
327 {
328     targz, tar, zip,
329 }
330 
331 private immutable(string[]) supportedArchiveExts = [
332     ".zip", ".tar.gz", ".tar"
333 ];
334 
335 private bool isSupportedArchiveExt(in string path)
336 {
337     import std.algorithm : endsWith;
338     import std.uni : toLower;
339 
340     const lpath = path.toLower;
341     return lpath.endsWith(".zip") || lpath.endsWith(".tar.gz") || lpath.endsWith(".tar");
342 }
343 
344 private ArchiveFormat archiveFormat(in string path)
345 {
346     import std.algorithm : endsWith;
347 
348     if (path.endsWith(".zip")) return ArchiveFormat.zip;
349     if (path.endsWith(".tar.gz")) return ArchiveFormat.targz;
350     if (path.endsWith(".tar")) return ArchiveFormat.tar;
351     assert(false);
352 }
353 
354 private string urlLastComp(in string url)
355 {
356     size_t ind = url.length - 1;
357     while (ind >= 0 && url[ind] != '/') {
358         ind--;
359     }
360     return url[ind+1 .. $];
361 }
362 
363 private string likelySrcDirName(in string archive)
364 {
365     import std.algorithm : endsWith;
366     import std.uni : toLower;
367 
368     assert(isSupportedArchiveExt(archive));
369 
370     foreach(ext; supportedArchiveExts) {
371         if (archive.toLower.endsWith(ext)) {
372             return archive[0 .. $-ext.length];
373         }
374     }
375     assert(false);
376 }
377 
378 unittest
379 {
380     assert(likelySrcDirName("/path/archivename.tar.gz") == "/path/archivename");
381 }