#!/usr/bin/php -qCddisable_functions=mysql_query
<?php
/**
* api: cli
* type: application
* title: apt-phparchive
* description: generates Packages.* and Release.* indexes for _trivial_ APT repository
* version: 0.7
* author: mario <mario#include-once:org>
* category: utility
* config:
* { name: KEYNAME, value: dpkg1, title: "GPG key identifier", description: "used as GPG key for signing the Packages.gpg file" }
* status: stable
*
*
* Traverses directories for common package files.
* Should index .deb .php .zip .exe .msi and possibly .rpm all at once.
*
* It's best invoked as:
*
* apt-phparchive generate .
*
* That creates "Packages" and "Packages.xz" and "Release", "Release.gpg"
* and "InRelease" entries. Requires installed Unix tools and GPG for signing.
*
* Arguments are only loosely modelled after "apt-ftparchive".
*
* Package types:
* - DEB archives are inspected with ar|gzip|tar
* - PHP plugins are searched for a comment block with key:value meta infos
according to http://milki.include-once.org/genericplugins/
* - ZIP files are assumed to contain a main PHP script, again with meta
* data comment block. (basename.zip == contained basename.php)
* - EXE and MSI binaries are searched for an appended debian control block.
* (Must be separated from the binary by a newline and followed by one.)
* - RPM archives are introspected with `rpm -qip`
*
* The field names are mapped onto the Debian and APT standard as follows:
*
* DEBIAN PHP PLUGINS RPM
*
* Package: id: Name:
* Version: version: Version:
* Maintainer: author: Packager:
* Section: category: Group:
* Description: description+help:
* Summary:
* title:
* api:
* Release:
* Copyright: license: License:
* homepage:
* type:
* url:
* Priority: priority:
* Depends: depends:
* config:
* Architecture: (all) Architecture:
*
* Other fields are just retained verbatim.
*
* CAVEATS:
* - The replacing of empty lines in Description: fields is a bit redundat
* (3 regexes where 1 might suffice), but better safe than sorry, and we
* have varying input formats and even deb files could be mispackaged.
* - Currently this tool only works in the current directory.
* - Simple getopt variant, unknown args just get ignored.
* - Don't bug me about global variables or your precious cargo cult
* programming feelings on @.
* - DEB extraction can fail if the contained control.tar.gz doesn't list
* the control entry as exactly "./control" with local dir prefix
* (incorrectly handmade .deb packages).
* - RPM converison tested with 4.8, might fail if the output prettyprinting
* got changed in other versions.
* - The PHP plugin meta data "standard" isn't really one. Its current
* usage is still in flux. Wordpress plugin files are pre-converted by the
* generic_pmd handler.
* - No idea how APT-RPM would actually require the Packages index. The
* current rewriting into standard APT fields: might possibly break it.
* - There's a apt_preferences penalty for fully populated "InRelease"
* indexes. Setting any of Label, Suite, Version fields raises an
* overwrite warning from apt. All badly documented.
*
*/
#-- init
if (!function_exists("str_getcsv")) { function str_getcsv($str) { return array_map("trim", explode(",", $str)); } }
include("pluginmetadata.php");
#-- some regex definitions
// matches a consecutive Debian package control block (RFC821 style key:value list) - but only at the file end
define("RX_RFC821_BLOCK", b"/(^\w[\w.-]*\w:.+\R(^ .*\R)*)++\R*\z/m");
// rpm -qip output lines "Field : Value\n"
define("RX_RPM_QIP_HEADER", b"/^(\w+(?: \w+)?)\s*:\s*(.+)/m");
// break up Key: value lines
define("RX_RFC821_SPLIT", b"/^(\w+[-.]?\w+)\s*:\s*(.+\n(^ .*\n)*)/m");
// finds text lines that don't start with a key: and aren't already escaped by a leading " " space
define("RX_INDENT_RAW_TEXT", b"/(?!\A)^(?!(?!http)\w+:| \S)/m");
// only matches first three text blocks, no empty lines or any starting with spaces (for Descripion: field)
define("RX_3_PARAGRAPHS", b"/\A((^\S.*\R)+\R+){1,3}/m");
// find all-space or empty lines
define("RX_EMPTY_LINES", b"/^\s*\\n/m");
#-- defaults
$KEYNAME = "include-once";
$output_fh = STDOUT;
$types = str_getcsv("php,zip,deb,rpm,exe,msi");
$command = "help";
$dir = ".";
define("HAS_RPM", strlen(`which rpm 2>/dev/null`));
#-- cmdline args
$argv0 = $_SERVER["argv"][0];
$args = new ArrayObject(array(
"help" => argv::opt("-h", "--help"),
"quiet" => argv::opt("-q", "--quiet"),
"key" => argv::param("-k", "--key"),
"types" => argv::param("-t", "--types", "--type"),
"output" => argv::param("-o", "-O", "--output"),
"cache" => argv::param("-c", "-d", "--db", "--cache"),
"words" => array_values(argv::files()),
), 2);
#print_r($args);
#-- apply options
if ($args->words) {
$command = strtr(($args->words[0]), ".", "_");
$dir = $args->words[1];
}
if ($args->output) {
$output_fh = fopen($args->output, "w");
}
if ($args->key) {
$KEYNAME = $args->key;
}
if ($args->types) {
$types = str_getcsv($args->types);
}
if ($args->help) {
$command = "help";
}
if ($args->cache) {
$cache = new cache($args->cache);
}
else {
$cache = new nocache(":memory:"); // missing pdo driver
}
#-- main -------------------------------
if (method_exists("main", $command)) {
call_user_func(array("main", $command));
}
else {
out("Unknown command '$command'.\n");
}
// available commands
class main {
#-- synopsis
function help() {
print <<<TEXT
Usage: apt-phparchive [options] command
Index Commands: packages ./
release ./
In-Place Write: generate ./
packages.bz2
release.gpg
apt-phparchive generates index files for trivial APT repositories. It
looks for .php plugin and .zip module packages, but also regular .deb
packages, .exe and .msi installers, and possibly .rpm archives.
It really only works within the current directory.
Options:
-h This help text
-k key GPG key identifier for signing (Release.gpg or Generate command)
-o file Output to file instead of stdout (for Packages and Release commands)
-t foo,bar List of searched package types ("php,zip,deb,rpm,exe,msi" per default)
--db file SQLite cache file\n\n
TEXT;
}
#-- create "Packages" file
function Packages() {
global $dir, $types, $cache;
// run over list of directories (nah, just one)
foreach (files(array($dir)) as $fn) {
$control = "";
$ext = pathinfo($fn, PATHINFO_EXTENSION);
#-- check if it's one of the searched types
if (!in_array($ext, $types)) {
{ continue; }
}
#-- check if scanned recently
elseif ($control = $cache->fetch($fn)) {
out("$control\n");
$control = ""; // do not regenerate checksums
}
#-- extract meta data
else switch ($ext) {
// fetch plugin meta info from comment block in php script
case "php":
$control = convert::php(file_get_contents($fn), $fn);
break;
// extract info from main php script contained in a zip
case "zip":
$control = convert::php(convert::zip($fn), $fn);
break;
// extracts control file to stdout
case "deb":
$_fn = escapeshellarg($fn);
$control = strval(`ar p $_fn control.tar.gz | tar xOz ./control`); // add " control 2>/dev/null" to also find broken packages
break;
// contains a simple appended Package: text block if it's one of our SFX files
case "exe":
case "msi":
$control = convert::binary_extract_rfc821($fn);
break;
// can only work with actual `rpm` tool
case "rpm":
$_fn = escapeshellarg($fn);
HAS_RPM and $control = convert::rpm(`rpm -qip $_fn`);
break;
default:
}
#-- did we get an control block?
if ($control) {
$data = file_get_contents($fn);
$file = "Filename: $fn\n"
. "Size: " . filesize($fn) . "\n";
$sums = "MD5sum: " . md5($data) . "\n"
. "SHA1: " . sha1($data) . "\n"
. "SHA256: " . hash("sha256", $data) . "\n";
#. "SHA512: " . hash("sha512", $data) . "\n";
out(
$cache->store($fn, rep_struct::normalize($file . $control . $sums)),
"\n"
);
}
}
}
#-- compress "Packages"
function Packages_bz2() {
global $dir;
out(
`bzip2 -kf $dir/Packages`
);
}
function Packages_xz() {
global $dir;
out(
`xz -kf $dir/Packages`
);
}
#-- create "Release" file
function Release() {
out(join("\n", array(
# "Origin: Ubuntu",
# "Label: $GLOBALS[KEYNAME]",
# "Suite: artful",
# "Version: 17.10",
"Date: " . gmdate(DATE_RFC822) . "",
# "Codename: artful\n",
"Architectures: amd64 i386 all",
"Components: universe multiverse",
"Description: $GLOBALS[KEYNAME]",
"NotAutomatic: no",
"ButAutomaticUpgrades: yes",
)) . "\n");
// iterate over checksum types and index files
foreach (array("MD5Sum"=>"md5", "SHA1"=>"sha1", "SHA256"=>"sha256", "RIPE160"=>"ripemd160") as $hashname=>$hash) {
out(
"$hashname:\n"
);
foreach ((glob("Packages*")+glob("Release*")) as $fn) {
// Not sure if we have to, but "Release" file gets ignored
if ($fn == "Release") {
$filesize = 0;
$chksum = hash($hash, "");
}
else {
$filesize = filesize($fn);
$chksum = hash_file($hash, $fn);
}
// fixed space output format
out(
" ", $chksum,
" ", str_pad($filesize, 16, " ", STR_PAD_LEFT),
" ", $fn, "\n"
);
}
}
out("\n");
}
#-- inline-signed Release file
# https://wiki.debian.org/DebianRepository/Format
function InRelease() {
global $dir, $KEYNAME;
out(`gpg --yes -s --clearsign --digest-algo SHA512 -u $KEYNAME -o /dev/stdout $dir/Release`);
}
#-- sign "Release.gpg"
function Release_gpg() {
global $dir, $KEYNAME;
out(
`gpg --yes -abs --digest-algo SHA512 -u $KEYNAME -o $dir/Release.gpg $dir/Release`
);
}
#-- perform all index generation things at once
function Generate() {
global $dir, $argv0, $args;
out(`$argv0 Packages $dir --cache {$args->cache} > $dir/Packages`);
# out(`$argv0 Packages.bz2 $dir`);
out(`$argv0 Packages.xz $dir`);
out(`$argv0 Release $dir > $dir/Release`);
out(`$argv0 Release.gpg $dir`);
out(`$argv0 InRelease $dir > $dir/InRelease`);
}
}//main
#-- utility code --
/**
* Fetch known commandline arguments.
*
*/
class argv {
// standalone -opt
function opt() {
$args = func_get_args();
return count(array_intersect($_SERVER["argv"], $args));
}
// parameter --arg VALUE
function param($spec) {
$spec = func_get_args();
foreach ($_SERVER["argv"] as $i=>$arg) {
if (in_array($arg, $spec)) return current(array_splice($_SERVER["argv"], $i+1, 1));
}
}
// anything that doesn't look like a parameter
function files() {
return preg_grep('/^[^-]/', array_slice($_SERVER["argv"], 1));
}
}//argv
/**
* Convert list of dirs into filename list.
*
* @return iterator
*/
function files($dir) {
return new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir[0]));
}
/**
* Write list of strings to stdout, or any open file handle.
*
*/
function out(/*...*/) {
global $output_fh;
$args = func_get_args();
fprintf($output_fh, join($args));
}
#-- repository generation --
/**
* Rewrite or normalize repository entry.
*
*/
class/*group*/ rep_struct {
/**
* Replaces empty lines (or just with spaces) with a " ." placeholder.
*
*/
function fix_emptylines($str) {
return preg_replace(RX_EMPTY_LINES, " .\n", $str);
}
/**
* Fix blank lines in Description: field, also cut it down to 3
* paragraphs.
*
*/
function description($help) {
if (preg_match(RX_3_PARAGRAPHS, trim($help), $m)) {
$help = $m[0]; // cut down to 3 paragraphs
}
$help = preg_replace("/^/m", " ", $help); // prepend spaces on each line
return rep_struct::fix_emptylines($help);
}
/**
* Join rfc821-style Key:Value list into a text block.
*
*/
function join_key_value_list($info) {
$out = "";
foreach ($info as $key=>$value) {
$out .= "$key: " . trim($value) . "\n";
}
return $out;
}
/**
* Extracts valid Lines: from a control block.
*
*/
function entry($control) {
preg_match_all(RX_RFC821_SPLIT, $control, $m);
return array_combine($m[1], $m[2]);
}
/**
* Reassembles control block, reorders entries.
*
*/
function normalize($control) {
// pre-fix, split
$control = rep_struct::fix_emptylines($control);
$entry = rep_struct::entry($control);
// move Description: entry
foreach (array("Description") as $key) {
if (isset($entry[$key])) {
$tmp = $entry[$key];
unset($entry[$key]);
$entry[$key] = $tmp;
}
}
// merge lines again
return rep_struct::join_key_value_list($entry);
}
}//rep_struct
#-- plugin meta data
/**
* Transform package types.
*
*/
class/*group*/ convert {
/**
* Extract PHP plugin meta block from filename.
*
*/
function php($src, $basename) {
// parse /*key:value*/ comment block
$p = new generic_pmd();
$info = @( $p->parse($src) );
$info["id"] = strtok(basename($basename), "-.");
$info["architecture"] = "all";
// convert and assemble into debian control block
if (convert::is_packageable_plugin($info)) {
$info = convert::rewrite_info($info);
return $out = rep_struct::join_key_value_list($info);
}
}
/**
* Filter for which packages are considered real packages.
*
*/
function is_packageable_plugin($info) {
return @(
TRUE
&& $info["version"] // must have at least version:, id: and title: fields
&& $info["id"]
&& $info["title"]
&& !in_array($info["type"], array("R", "ignore")) // ignore faux plugin types
&& !in_array($info["api"], array("")) // ignore api: types
&& $info["package"] != "false"
);
}
/**
* Transpose PHP plugin standard field: names onto Debian: control keys.
*
*/
function rewrite_info($info) {
$out = array();
$map = array(
"id" => "Package",
"author" => "Maintainer",
"category" => "Section",
);
#-- merge title: and help: into Description:, but keep separate Title:
$info["description"] = @trim($info["description"] . "\n" . rep_struct::description($info["help"]));
unset($info["help"]);
#-- map
foreach ($info as $key=>$value) {
if (isset($map[$key])) {
$key = $map[$key];
}
$key = ucwords($key);
$value = preg_replace(RX_INDENT_RAW_TEXT, " ", $value); // indent raw text lines
$out[$key] = $value;
}
return $out;
}
/**
* Get entry from zip file, whose name matches "basename.php"
*
*/
function zip($fn) {
$base = strtok(basename($fn), "-.");
$z = new ZipArchive;
$z->open($fn);
if ($i = $z->locateName("$base.php")) {
return $z->getFromIndex($i);
}
else {
return $z->getFromIndex(0);
}
}
/**
* Convert RPM print output into field list. Rewrite some entries.
*
*/
function rpm($text) {
// split
list($header, $body) = explode("\n\n", $text, 2);
preg_match_all(RX_RPM_QIP_HEADER, $header, $match);
$info = array_combine($match[1], $match[2]);
// rewrite fields
$map = array(
"Name" => "Package",
"Packager" => "Maintainer",
"Group" => "Section",
"Vendor" => "Author",
"Summary" => "Title",
);
foreach ($info as $key=>$value) {
$key = strtr($key, " ", "-");
$key = isset($map[$key]) ? $map[$key] : $key;
$control[$key] = $value;
}
$control["Description"] = rep_struct::description(trim($body));
// assemble
return
$out = rep_struct::join_key_value_list($control);
}
/**
* Read in binary file, check at end for a block of Key:value lines.
*
*/
function binary_extract_rfc821($fn) {
if ($binary = file_get_contents($fn)) {
if (preg_match(RX_RFC821_BLOCK, $binary, $m)) {
$text = $m[0];
$text = preg_replace("/\R/", "\n", $text);
$text = "Architecture: win32\n" . $text;
return $text;
}
}
}
}//convert
#-- cache --
/**
* Stores control block entries in a cache file.
*
*/
class cache extends PDO {
function __construct($fn=":memory:") {
parent::__construct("sqlite:$fn");
$this->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
$this->query("CREATE TABLE IF NOT EXISTS cache (fn VARCHAR PRIMARY KEY, time INT, control TEXT)");
}
function fetch($fn) {
$s = $this->prepare("SELECT control FROM cache WHERE fn=? AND time=?")
and $r = $s->execute(array($fn, filemtime($fn)))
and $o = $s->fetch();
return $o ? $o["control"] : NULL;
}
function store($fn, $data) {
$this->prepare("INSERT OR REPLACE INTO cache (fn, time, control) VALUES (?,?,?)")
->execute(array($fn, filemtime($fn), $data));
return $data;
}
}
class nocache { // fallback just in case if PDO or sqlite interface are missing
function fetch($fn) { return NULL; }
function store($fn, $data) { return $data; }
}
// if you are super concerned about closetags, use `phptags --unclosed` (or simpler `--whitespace` for fixing the actual issues, not the umbrage)
?>