001 /*
002 * Copyright 2010 The Apache Software Foundation.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.vafer.jdeb;
017
018 import java.io.ByteArrayInputStream;
019 import java.io.File;
020 import java.io.FileInputStream;
021 import java.io.FileNotFoundException;
022 import java.io.FileOutputStream;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.OutputStream;
026 import java.math.BigInteger;
027 import java.security.DigestOutputStream;
028 import java.security.MessageDigest;
029 import java.security.NoSuchAlgorithmException;
030 import java.text.ParseException;
031 import java.text.SimpleDateFormat;
032 import java.util.ArrayList;
033 import java.util.Arrays;
034 import java.util.Date;
035 import java.util.List;
036 import java.util.Locale;
037 import java.util.zip.GZIPOutputStream;
038
039 import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
040 import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream;
041 import org.apache.tools.bzip2.CBZip2OutputStream;
042 import org.apache.tools.tar.TarEntry;
043 import org.apache.tools.tar.TarOutputStream;
044 import org.vafer.jdeb.changes.ChangeSet;
045 import org.vafer.jdeb.changes.ChangesProvider;
046 import org.vafer.jdeb.descriptors.ChangesDescriptor;
047 import org.vafer.jdeb.descriptors.InvalidDescriptorException;
048 import org.vafer.jdeb.descriptors.PackageDescriptor;
049 import org.vafer.jdeb.mapping.PermMapper;
050 import org.vafer.jdeb.signing.SigningUtils;
051 import org.vafer.jdeb.utils.InformationOutputStream;
052 import org.vafer.jdeb.utils.Utils;
053 import org.vafer.jdeb.utils.VariableResolver;
054
055 /**
056 * The processor does the actual work of building the deb related files.
057 * It is been used by the ant task and (later) the maven plugin.
058 *
059 * @author Torsten Curdt <tcurdt@vafer.org>
060 */
061 public class Processor {
062
063 private final Console console;
064 private final VariableResolver resolver;
065
066 private static final class Total {
067 private BigInteger count = BigInteger.valueOf(0);
068
069 public void add(long size) {
070 count = count.add(BigInteger.valueOf(size));
071 }
072
073 public String toString() {
074 return "" + count;
075 }
076
077 // public BigInteger toBigInteger() {
078 // return count;
079 // }
080 }
081
082 public Processor( final Console pConsole, final VariableResolver pResolver ) {
083 console = pConsole;
084 resolver = pResolver;
085 }
086
087 private void addTo( final ArArchiveOutputStream pOutput, final String pName, final String pContent ) throws IOException {
088 final byte[] content = pContent.getBytes();
089 pOutput.putArchiveEntry(new ArArchiveEntry(pName, content.length));
090 pOutput.write(content);
091 pOutput.closeArchiveEntry();
092 }
093
094 private void addTo( final ArArchiveOutputStream pOutput, final String pName, final File pContent ) throws IOException {
095 pOutput.putArchiveEntry(new ArArchiveEntry(pName, pContent.length()));
096
097 final InputStream input = new FileInputStream(pContent);
098 try {
099 Utils.copy(input, pOutput);
100 } finally {
101 input.close();
102 }
103
104 pOutput.closeArchiveEntry();
105 }
106
107 /**
108 * Create the debian archive with from the provided control files and data producers.
109 *
110 * @param pControlFiles
111 * @param pData
112 * @param pOutput
113 * @param compression the compression method used for the data file (gzip, bzip2 or anything else for no compression)
114 * @return PackageDescriptor
115 * @throws PackagingException
116 */
117 public PackageDescriptor createDeb( final File[] pControlFiles, final DataProducer[] pData, final File pOutput, String compression ) throws PackagingException, InvalidDescriptorException {
118
119 File tempData = null;
120 File tempControl = null;
121
122 try {
123 tempData = File.createTempFile("deb", "data");
124 tempControl = File.createTempFile("deb", "control");
125
126 console.println("Building data");
127 final StringBuffer md5s = new StringBuffer();
128 final BigInteger size = buildData(pData, tempData, md5s, compression);
129
130 console.println("Building control");
131 final PackageDescriptor packageDescriptor = buildControl(pControlFiles, size, md5s, tempControl);
132
133 if (!packageDescriptor.isValid()) {
134 throw new InvalidDescriptorException(packageDescriptor);
135 }
136
137 pOutput.getParentFile().mkdirs();
138 final InformationOutputStream output = new InformationOutputStream(new FileOutputStream(pOutput), MessageDigest.getInstance("MD5"));
139
140 final ArArchiveOutputStream ar = new ArArchiveOutputStream(output);
141
142 addTo(ar, "debian-binary", "2.0\n");
143 addTo(ar, "control.tar.gz", tempControl);
144 addTo(ar, "data.tar" + getExtension(compression), tempData);
145
146 ar.close();
147
148 // intermediate values
149 packageDescriptor.set("MD5", output.getMd5());
150 packageDescriptor.set("Size", "" + output.getSize());
151 packageDescriptor.set("File", pOutput.getName());
152
153 return packageDescriptor;
154
155 } catch(InvalidDescriptorException e) {
156 throw e;
157 } catch(Exception e) {
158 throw new PackagingException("Could not create deb package", e);
159 } finally {
160 if (tempData != null) {
161 if (!tempData.delete()) {
162 throw new PackagingException("Could not delete " + tempData);
163 }
164 }
165 if (tempControl != null) {
166 if (!tempControl.delete()) {
167 throw new PackagingException("Could not delete " + tempControl);
168 }
169 }
170 }
171 }
172
173 /**
174 * Return the extension of a file compressed with the specified method.
175 *
176 * @param pCompression the compression method used
177 * @return
178 */
179 private String getExtension( final String pCompression ) {
180 if ("gzip".equals(pCompression)) {
181 return ".gz";
182 } else if ("bzip2".equals(pCompression)) {
183 return ".bz2";
184 } else {
185 return "";
186 }
187 }
188
189 /**
190 * Create changes file based on the provided PackageDescriptor.
191 * If pRing, pKey and pPassphrase are provided the changes file will also be signed.
192 * It returns a ChangesDescriptor reflecting the changes
193 * @param pPackageDescriptor
194 * @param pChangesProvider
195 * @param pRing
196 * @param pKey
197 * @param pPassphrase
198 * @param pOutput
199 * @return ChangesDescriptor
200 * @throws IOException
201 */
202 public ChangesDescriptor createChanges( final PackageDescriptor pPackageDescriptor, final ChangesProvider pChangesProvider, final InputStream pRing, final String pKey, final String pPassphrase, final OutputStream pOutput ) throws IOException, InvalidDescriptorException {
203
204 final ChangeSet[] changeSets = pChangesProvider.getChangesSets();
205 final ChangesDescriptor changesDescriptor = new ChangesDescriptor(pPackageDescriptor, changeSets);
206
207 changesDescriptor.set("Format", "1.7");
208
209 if (changesDescriptor.get("Binary") == null) {
210 changesDescriptor.set("Binary", changesDescriptor.get("Package"));
211 }
212
213 if (changesDescriptor.get("Source") == null) {
214 changesDescriptor.set("Source", changesDescriptor.get("Package"));
215 }
216
217 if (changesDescriptor.get("Description") == null) {
218 changesDescriptor.set("Description", "update to " + changesDescriptor.get("Version"));
219 }
220
221 final StringBuffer files = new StringBuffer("\n");
222 files.append(' ').append(changesDescriptor.get("MD5"));
223 files.append(' ').append(changesDescriptor.get("Size"));
224 files.append(' ').append(changesDescriptor.get("Section"));
225 files.append(' ').append(changesDescriptor.get("Priority"));
226 files.append(' ').append(changesDescriptor.get("File"));
227 changesDescriptor.set("Files", files.toString());
228
229 if (!changesDescriptor.isValid()) {
230 throw new InvalidDescriptorException(changesDescriptor);
231 }
232
233 final String changes = changesDescriptor.toString();
234 //console.println(changes);
235
236 final byte[] changesBytes = changes.getBytes("UTF-8");
237
238 if (pRing == null || pKey == null || pPassphrase == null) {
239 pOutput.write(changesBytes);
240 pOutput.close();
241 return changesDescriptor;
242 }
243
244 console.println("Signing changes with key " + pKey);
245
246 final InputStream input = new ByteArrayInputStream(changesBytes);
247
248 try {
249 SigningUtils.clearSign(input, pRing, pKey, pPassphrase, pOutput);
250 } catch (Exception e) {
251 e.printStackTrace();
252 }
253
254 pOutput.close();
255
256 return changesDescriptor;
257 }
258
259 /**
260 * Build control archive of the deb
261 * @param pControlFiles
262 * @param pDataSize
263 * @param pChecksums
264 * @param pOutput
265 * @return
266 * @throws FileNotFoundException
267 * @throws IOException
268 * @throws ParseException
269 */
270 private PackageDescriptor buildControl( final File[] pControlFiles, final BigInteger pDataSize, final StringBuffer pChecksums, final File pOutput ) throws IOException, ParseException {
271
272 PackageDescriptor packageDescriptor = null;
273
274 final TarOutputStream outputStream = new TarOutputStream(new GZIPOutputStream(new FileOutputStream(pOutput)));
275 outputStream.setLongFileMode(TarOutputStream.LONGFILE_GNU);
276
277 for (int i = 0; i < pControlFiles.length; i++) {
278 final File file = pControlFiles[i];
279
280 if (file.isDirectory()) {
281 continue;
282 }
283
284 final TarEntry entry = new TarEntry(file);
285
286 final String name = file.getName();
287
288 entry.setName("./" + name);
289 entry.setNames("root", "root");
290 entry.setMode(PermMapper.toMode("755"));
291
292 if ("control".equals(name)) {
293 packageDescriptor = new PackageDescriptor(new FileInputStream(file), resolver);
294
295 if (packageDescriptor.get("Date") == null) {
296 SimpleDateFormat fmt = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z", Locale.ENGLISH); // Mon, 26 Mar 2007 11:44:04 +0200 (RFC 2822)
297 // FIXME Is this field allowed in package descriptors ?
298 packageDescriptor.set("Date", fmt.format(new Date()));
299 }
300
301 if (packageDescriptor.get("Distribution") == null) {
302 packageDescriptor.set("Distribution", "unknown");
303 }
304
305 if (packageDescriptor.get("Urgency") == null) {
306 packageDescriptor.set("Urgency", "low");
307 }
308
309 final String debFullName = System.getenv("DEBFULLNAME");
310 final String debEmail = System.getenv("DEBEMAIL");
311
312 if (debFullName != null && debEmail != null) {
313 packageDescriptor.set("Maintainer", debFullName + " <" + debEmail + ">");
314 console.println("Using maintainer from the environment variables.");
315 }
316
317 continue;
318 }
319
320 final InputStream inputStream = new FileInputStream(file);
321
322 outputStream.putNextEntry(entry);
323
324 Utils.copy(inputStream, outputStream);
325
326 outputStream.closeEntry();
327
328 inputStream.close();
329
330 }
331
332 if (packageDescriptor == null) {
333 throw new FileNotFoundException("No control file in " + Arrays.toString(pControlFiles));
334 }
335
336 packageDescriptor.set("Installed-Size", pDataSize.divide(BigInteger.valueOf(1024)).toString());
337
338 addEntry("control", packageDescriptor.toString(), outputStream);
339
340 addEntry("md5sums", pChecksums.toString(), outputStream);
341
342 outputStream.close();
343
344 return packageDescriptor;
345 }
346
347 /**
348 * Build the data archive of the deb from the provided DataProducers
349 * @param pData
350 * @param pOutput
351 * @param pChecksums
352 * @param pCompression the compression method used for the data file (gzip, bzip2 or anything else for no compression)
353 * @return
354 * @throws NoSuchAlgorithmException
355 * @throws IOException
356 */
357 BigInteger buildData( final DataProducer[] pData, final File pOutput, final StringBuffer pChecksums, String pCompression ) throws NoSuchAlgorithmException, IOException {
358
359 OutputStream out = new FileOutputStream(pOutput);
360 if ("gzip".equals(pCompression)) {
361 out = new GZIPOutputStream(out);
362 } else if ("bzip2".equals(pCompression)) {
363 out.write("BZ".getBytes());
364 out = new CBZip2OutputStream(out);
365 }
366
367 final TarOutputStream outputStream = new TarOutputStream(out);
368 outputStream.setLongFileMode(TarOutputStream.LONGFILE_GNU);
369
370 final MessageDigest digest = MessageDigest.getInstance("MD5");
371
372 final Total dataSize = new Total();
373
374 final List addedDirectories = new ArrayList();
375 final DataConsumer receiver = new DataConsumer() {
376 public void onEachDir( String dirname, String linkname, String user, int uid, String group, int gid, int mode, long size ) throws IOException {
377 dirname = fixPath(dirname);
378
379 createParentDirectories((new File(dirname)).getParent(), user, uid, group, gid);
380
381 // The directory passed in explicitly by the caller also gets the passed-in mode. (Unlike
382 // the parent directories for now. See related comments at "int mode =" in
383 // createParentDirectories, including about a possible bug.)
384 createDirectory(dirname, user, uid, group, gid, mode, 0);
385
386 console.println("dir: " + dirname);
387 }
388
389 public void onEachFile( InputStream inputStream, String filename, String linkname, String user, int uid, String group, int gid, int mode, long size ) throws IOException {
390 filename = fixPath(filename);
391
392 createParentDirectories((new File(filename)).getParent(), user, uid, group, gid);
393
394 TarEntry entry = new TarEntry(filename);
395
396 // FIXME: link is in the constructor
397 entry.setUserName(user);
398 entry.setUserId(uid);
399 entry.setGroupName(group);
400 entry.setGroupId(gid);
401 entry.setMode(mode);
402 entry.setSize(size);
403
404 outputStream.putNextEntry(entry);
405
406 dataSize.add(size);
407
408 digest.reset();
409
410 Utils.copy(inputStream, new DigestOutputStream(outputStream, digest));
411
412 final String md5 = Utils.toHex(digest.digest());
413
414 outputStream.closeEntry();
415
416 console.println(
417 "file:" + entry.getName() +
418 " size:" + entry.getSize() +
419 " mode:" + entry.getMode() +
420 " linkname:" + entry.getLinkName() +
421 " username:" + entry.getUserName() +
422 " userid:" + entry.getUserId() +
423 " groupname:" + entry.getGroupName() +
424 " groupid:" + entry.getGroupId() +
425 " modtime:" + entry.getModTime() +
426 " md5: " + md5
427 );
428
429 pChecksums.append(md5).append(" ").append(entry.getName()).append('\n');
430
431 }
432
433 private String fixPath(String path) {
434 // If we're receiving directory names from Windows, then we'll convert to use slash
435 // This does eliminate the ability to use of a backslash in a directory name on *NIX, but in practice, this is a non-issue
436 if (path.indexOf('\\') > -1) {
437 path = path.replace('\\', '/');
438 }
439 // ensure the path is like : ./foo/bar
440 if (path.startsWith("/")) {
441 path = "." + path;
442 } else if (!path.startsWith("./")) {
443 path = "./" + path;
444 }
445 return path;
446 }
447
448 private void createDirectory(String directory, String user, int uid, String group, int gid, int mode, long size) throws IOException {
449 // All dirs should end with "/" when created, or the test DebAndTaskTestCase.testTarFileSet() thinks its a file
450 // and so thinks it has the wrong permission.
451 // This consistency also helps when checking if a directory already exists in addedDirectories.
452
453 if (!directory.endsWith("/")) {
454 directory += "/";
455 }
456
457 if (!addedDirectories.contains(directory)) {
458 TarEntry entry = new TarEntry(directory);
459 // FIXME: link is in the constructor
460 entry.setUserName(user);
461 entry.setUserId(uid);
462 entry.setGroupName(group);
463 entry.setGroupId(gid);
464 entry.setMode(mode);
465 entry.setSize(size);
466
467 outputStream.putNextEntry(entry);
468 outputStream.closeEntry();
469 addedDirectories.add(directory); // so addedDirectories consistently have "/" for finding duplicates.
470 }
471 }
472
473 private void createParentDirectories(String dirname, String user, int uid, String group, int gid) throws IOException {
474 // Debian packages must have parent directories created
475 // before sub-directories or files can be installed.
476 // For example, if an entry of ./usr/lib/foo/bar existed
477 // in a .deb package, but the ./usr/lib/foo directory didn't
478 // exist, the package installation would fail. The .deb must
479 // then have an entry for ./usr/lib/foo and then ./usr/lib/foo/bar
480
481 if (dirname == null) {
482 return;
483 }
484
485 // The loop below will create entries for all parent directories
486 // to ensure that .deb packages will install correctly.
487 String[] pathParts = dirname.split("\\/");
488 String parentDir = "./";
489 for (int i = 1; i < pathParts.length; i++) {
490 parentDir += pathParts[i] + "/";
491 // Make it so the dirs can be traversed by users.
492 // We could instead try something more granular, like setting the directory
493 // permission to 'rx' for each of the 3 user/group/other read permissions
494 // found on the file being added (ie, only if "other" has read
495 // permission on the main node, then add o+rx permission on all the containing
496 // directories, same w/ user & group), and then also we'd have to
497 // check the parentDirs collection of those already added to
498 // see if those permissions need to be similarly updated. (Note, it hasn't
499 // been demonstrated, but there might be a bug if a user specifically
500 // requests a directory with certain permissions,
501 // that has already been auto-created because it was a parent, and if so, go set
502 // the user-requested mode on that directory instead of this automatic one.)
503 // But for now, keeping it simple by making every dir a+rx. Examples are:
504 // drw-r----- fs/fs # what you get with setMode(mode)
505 // drwxr-xr-x fs/fs # Usable. Too loose?
506 int mode = TarEntry.DEFAULT_DIR_MODE;
507
508 createDirectory(parentDir, user, uid, group, gid, mode, 0);
509 }
510 }
511 };
512
513 for (int i = 0; i < pData.length; i++) {
514 final DataProducer data = pData[i];
515 data.produce(receiver);
516 }
517
518 outputStream.close();
519
520 console.println("Total size: " + dataSize);
521
522 return dataSize.count;
523 }
524
525 private static void addEntry( final String pName, final String pContent, final TarOutputStream pOutput ) throws IOException {
526 final byte[] data = pContent.getBytes("UTF-8");
527
528 final TarEntry entry = new TarEntry("./" + pName);
529 entry.setSize(data.length);
530 entry.setNames("root", "root");
531
532 pOutput.putNextEntry(entry);
533 pOutput.write(data);
534 pOutput.closeEntry();
535 }
536
537
538 }