Collection of mostly command line tools / PHP scripts. Somewhat out of date.

⌈⌋ ⎇ branch:  scripts + snippets


Artifact [64a385f976]

Artifact 64a385f97601da5db4d81d4e74b4aa436f84c543:

  • Executable file apt-phparchive — part of check-in [beb19b64e0] at 2012-01-09 04:13:02 on branch trunk — functioning version of apt-phparchive (user: mario size: 18592)

     1
     2
     3
     4
     5
     6
     7
     8
     9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
   100
   101
   102
   103
   104
   105
   106
   107
   108
   109
   110
   111
   112
   113
   114
   115
   116
   117
   118
   119
   120
   121
   122
   123
   124
   125
   126
   127
   128
   129
   130
   131
   132
   133
   134
   135
   136
   137
   138
   139
   140
   141
   142
   143
   144
   145
   146
   147
   148
   149
   150
   151
   152
   153
   154
   155
   156
   157
   158
   159
   160
   161
   162
   163
   164
   165
   166
   167
   168
   169
   170
   171
   172
   173
   174
   175
   176
   177
   178
   179
   180
   181
   182
   183
   184
   185
   186
   187
   188
   189
   190
   191
   192
   193
   194
   195
   196
   197
   198
   199
   200
   201
   202
   203
   204
   205
   206
   207
   208
   209
   210
   211
   212
   213
   214
   215
   216
   217
   218
   219
   220
   221
   222
   223
   224
   225
   226
   227
   228
   229
   230
   231
   232
   233
   234
   235
   236
   237
   238
   239
   240
   241
   242
   243
   244
   245
   246
   247
   248
   249
   250
   251
   252
   253
   254
   255
   256
   257
   258
   259
   260
   261
   262
   263
   264
   265
   266
   267
   268
   269
   270
   271
   272
   273
   274
   275
   276
   277
   278
   279
   280
   281
   282
   283
   284
   285
   286
   287
   288
   289
   290
   291
   292
   293
   294
   295
   296
   297
   298
   299
   300
   301
   302
   303
   304
   305
   306
   307
   308
   309
   310
   311
   312
   313
   314
   315
   316
   317
   318
   319
   320
   321
   322
   323
   324
   325
   326
   327
   328
   329
   330
   331
   332
   333
   334
   335
   336
   337
   338
   339
   340
   341
   342
   343
   344
   345
   346
   347
   348
   349
   350
   351
   352
   353
   354
   355
   356
   357
   358
   359
   360
   361
   362
   363
   364
   365
   366
   367
   368
   369
   370
   371
   372
   373
   374
   375
   376
   377
   378
   379
   380
   381
   382
   383
   384
   385
   386
   387
   388
   389
   390
   391
   392
   393
   394
   395
   396
   397
   398
   399
   400
   401
   402
   403
   404
   405
   406
   407
   408
   409
   410
   411
   412
   413
   414
   415
   416
   417
   418
   419
   420
   421
   422
   423
   424
   425
   426
   427
   428
   429
   430
   431
   432
   433
   434
   435
   436
   437
   438
   439
   440
   441
   442
   443
   444
   445
   446
   447
   448
   449
   450
   451
   452
   453
   454
   455
   456
   457
   458
   459
   460
   461
   462
   463
   464
   465
   466
   467
   468
   469
   470
   471
   472
   473
   474
   475
   476
   477
   478
   479
   480
   481
   482
   483
   484
   485
   486
   487
   488
   489
   490
   491
   492
   493
   494
   495
   496
   497
   498
   499
   500
   501
   502
   503
   504
   505
   506
   507
   508
   509
   510
   511
   512
   513
   514
   515
   516
   517
   518
   519
   520
   521
   522
   523
   524
   525
   526
   527
   528
   529
   530
   531
   532
   533
   534
   535
   536
   537
   538
   539
   540
   541
   542
   543
   544
   545
   546
   547
   548
   549
   550
   551
   552
   553
   554
   555
   556
   557
   558
   559
   560
   561
   562
   563
   564
   565
   566
   567
   568
   569
   570
   571
   572
   573
   574
   575
   576
   577
   578
   579
   580
   581
   582
   583
   584
   585
   586
   587
   588
   589
   590
   591
   592
   593
   594
   595
   596
   597
   598
   599
   600
   601
   602
   603
   604
   605
   606
   607
   608
   609
   610
   611
   612
   613
   614
   615
   616
   617
   618
   619
   620
   621
   622
   623
   624
   625
   626
   627
   628
   629
   630
   631
   632
   633
   634
   635
   636
   637
   638
   639
   640
   641
   642
   643
   644
   645
   646
   647
   648
   649
   650
   651
   652
   653
   654
#!/usr/bin/php -qCddisable_functions=mysql_query
<?php
/**
 * api: cli
 * type: application
 * title: apt-phparchive
 * description: generates Packages.* and Release.* indexes for _trivial_ APT repository
 * version: 0.6
 * author: mario <mario#include-once:org>
 * category: utility
 * config: <var name="KEYNAME" value="dpkg1" title="GPG key identifier" description="used as GPG key for signing the Packages.gpg file" />
 *
 *
 * Traverses directories for common package files.
 * Should index .deb .php .zip .exe .msi and possibly .rpm all at once.
 *
 *
 * It's best invoked as:
 *
 *     apt-phparchive generate .
 *
 * That creates "Packages" and "Packages.bz2" and "Release" and "Release.gpg"
 * entries. Requires installed Unix tools and GPG for signing.
 *
 * Arguments are only loosely modelled after "apt-ftparchive".
 *
 * Package types:
 * - DEB archives are inspected with ar|gzip|tar
 * - PHP plugins are searched for a comment block with key:value meta infos
     according to http://milki.include-once.org/genericplugins/
 * - ZIP files are assumed to contain a main PHP script, again with meta
 *   data comment block. (basename.zip == contained basename.php)
 * - EXE and MSI binaries are searched for an appended debian control block.
 *   (Must be separated from the binary by a newline and followed by one.)
 * - RPM archives are introspected with `rpm -qip`
 *
 * The field names are mapped onto the Debian and APT standard as follows:
 *
 *    DEBIAN          PHP PLUGINS        RPM
 *
 *    Package:        id:                Name:
 *    Version:        version:           Version:
 *    Maintainer:     author:            Packager:
 *    Section:        category:          Group:
 *    Description:    description+help:
 *                                       Summary:
 *                    title:
 *                    api:
 *                                       Release:
 *    Copyright:      license:           License:
 *                    homepage:
 *                    type:
 *                    url:
 *    Priority:       priority:
 *    Depends:        depends:
 *                    config:
 *    Architecture:   (all)              Architecture:
 *
 * Other fields are just retained verbatim.
 *
 * CAVEATS:
 * - The replacing of empty lines in Description: fields is a bit redundat
 *   (3 regexes where 1 might suffice), but better safe than sorry, and we
 *   have varying input formats and even deb files could be mispackaged.
 * - Currently this tool only works in the current directory.
 * - Simple getopt variant, unknown args just get ignored.
 * - Don't bug me about global variables or your precious cargo cult
 *   programming feelings on @.
 * - DEB extraction can fail if the contained control.tar.gz doesn't list
 *   the control entry as exactly "./control" with local dir prefix
 *   (incorrectly handmade .deb packages).
 * - RPM converison tested with 4.8, might fail if the output prettyprinting
 *   got changed in other versions.
 * - The PHP plugin meta data "standard" isn't really one. Its current
 *   usage is still in flux. Wordpress plugin files are pre-converted by the
 *   generic_pmd handler.
 * - No idea how APT-RPM would actually require the Packages index. The
 *   current rewriting into standard APT fields: might possibly break it.
 *
 */

#-- init
if (!function_exists("str_getcsv")) { function str_getcsv($str) { return array_map("trim", explode(",", $str)); } }
include("pluginmetadata.php");

#-- some regex definitions
  // matches a consecutive Debian package control block (RFC821 style key:value list) - but only at the file end
define("RX_RFC821_BLOCK", b"/(^\w[\w.-]*\w:.+\R(^ .*\R)*)++\R*\z/m");
  // rpm -qip output lines "Field    : Value\n"
define("RX_RPM_QIP_HEADER", b"/^(\w+(?: \w+)?)\s*:\s*(.+)/m");
  // break up Key: value lines
define("RX_RFC821_SPLIT", b"/^(\w+[-.]?\w+)\s*:\s*(.+\n(^ .*\n)*)/m");
  // finds text lines that don't start with a key: and aren't already escaped by a leading " " space
define("RX_INDENT_RAW_TEXT", b"/(?!\A)^(?!(?!http)\w+:| \S)/m");
  // only matches first three text blocks, no empty lines or any starting with spaces (for Descripion: field)
define("RX_3_PARAGRAPHS", b"/\A((^\S.*\R)+\R+){1,3}/m");
  // find all-space or empty lines
define("RX_EMPTY_LINES", b"/^\s*\\n/m");


#-- defaults
$KEYNAME = "include-once";
$output_fh = STDOUT;
$types = str_getcsv("php,zip,deb,rpm,exe,msi");
$command = "help";
$dir = ".";
define("HAS_RPM", strlen(`which rpm 2>/dev/null`));


#-- cmdline args
$argv0 = $_SERVER["argv"][0];
$args = new ArrayObject(array(
    "help" => argv::opt("-h", "--help"),
    "quiet" => argv::opt("-q", "--quiet"),
    "key" => argv::param("-k", "--key"),
    "types" => argv::param("-t", "--types", "--type"),
    "output" => argv::param("-o", "-O", "--output"),
    "cache" => argv::param("-c", "-d", "--db", "--cache"),
    "words" => array_values(argv::files()),
), 2);
#print_r($args);


#-- apply options
if ($args->words) {
    $command = strtr(($args->words[0]), ".", "_");
    $dir = $args->words[1];
}
if ($args->output) {
    $output_fh = fopen($args->output, "w");
}
if ($args->key) {
    $KEYNAME = $args->key;
}
if ($args->types) {
    $types = str_getcsv($args->types);
}
if ($args->help) {
    $command = "help";
}
if ($args->cache) {
    $cache = new cache($args->cache);
}
else {
    $cache = new nocache(":memory:");  // missing pdo driver
}



#-- main -------------------------------
if (method_exists("main", $command)) {
    call_user_func(array("main", $command));
}
else {
    out("Unknown command '$command'.\n");
}




// available commands
class main {


    #-- synopsis
    function help() {
        print <<<TEXT
Usage: apt-phparchive [options] command
Index Commands: packages ./
                release ./
In-Place Write: generate ./
                packages.bz2  
                release.gpg

apt-phparchive generates index files for trivial APT repositories. It
looks for .php plugin and .zip module packages, but also regular .deb
packages, .exe and .msi installers, and possibly .rpm archives.

It really only works within the current directory.

Options:
  -h          This help text
  -k key      GPG key identifier for signing (Release.gpg or Generate command)
  -o file     Output to file instead of stdout (for Packages and Release commands)
  -t foo,bar  List of searched package types ("php,zip,deb,rpm,exe,msi" per default)
  --db file   SQLite cache file\n\n
TEXT;
    }


    #-- create "Packages" file
    function Packages() {
        global $dir, $types, $cache;

        // run over list of directories (nah, just one)
        foreach (files(array($dir)) as $fn) {
           $control = "";
           $ext = pathinfo($fn, PATHINFO_EXTENSION);


           #-- check if it's one of the searched types
           if (!in_array($ext, $types)) {
               { continue; }
           }
           
           #-- check if scanned recently
           elseif ($control = $cache->fetch($fn)) {
               out("$control\n");
               $control = "";  // do not regenerate checksums
           }
           
           #-- extract meta data
           else switch ($ext) {

               // fetch plugin meta info from comment block in php script
               case "php":
                  $control = convert::php(file_get_contents($fn), $fn);
                  break;

               // extract info from main php script contained in a zip
               case "zip":
                  $control = convert::php(convert::zip($fn), $fn);
                  break;

               // extracts control file to stdout
               case "deb":
                  $_fn = escapeshellarg($fn);
                  $control = strval(`ar p $_fn control.tar.gz | tar xOz ./control`); // add " control 2>/dev/null" to also find broken packages
                  break;

               // contains a simple appended Package: text block if it's one of our SFX files
               case "exe":
               case "msi":
                  $control = convert::binary_extract_rfc821($fn);
                  break;

               // can only work with actual `rpm` tool
               case "rpm":
                  $_fn = escapeshellarg($fn);
                  HAS_RPM and $control = convert::rpm(`rpm -qip $_fn`);
                  break;

               default:
           }


           #-- did we get an control block?
           if ($control) {
               $data = file_get_contents($fn);
               $file = "Filename: $fn\n"
                     . "Size: " . filesize($fn) . "\n";
               $sums = "MD5sum: " . md5($data) . "\n"
                     . "SHA1: " . sha1($data) . "\n"
                     . "SHA256: " . hash("sha256", $data) . "\n";
                    #. "SHA512: " . hash("sha512", $data) . "\n";
               out(
                   $cache->store($fn, rep_struct::normalize($file . $control . $sums)),
                   "\n"
               );
           }
        }
    }


    #-- compress "Packages"
    function Packages_bz2() {
        global $dir;
        out( 
            `bzip2 -kf $dir/Packages`
        );
    }


    #-- create "Release" file
    function Release() {
        out(
            "Date: " . gmdate(DATE_RFC822) . "\n"
        );

        // iterate over checksum types and index files
        foreach (array("MD5Sum"=>"md5", "SHA1"=>"sha1", "SHA256"=>"sha256", "RIPE160"=>"ripemd160") as $hashname=>$hash) {
            out(
                "$hashname:\n"
            );
            foreach (array("Packages", "Packages.bz2", "Release") as $fn) {

                // Not sure if we have to, but "Release" file gets ignored
                if ($fn == "Release") {
                    $filesize = 0;
                    $chksum = hash($hash, "");
                }
                else {
                    $filesize = filesize($fn);
                    $chksum = hash_file($hash, $fn);
                }

                // fixed space output format
                out(
                    " ", $chksum,
                    " ", str_pad($filesize, 16, " ", STR_PAD_LEFT),
                    " ", $fn, "\n"
                );
            }
        }
        out("\n");
    }



    #-- sign "Release.gpg"
    function Release_gpg() {
        global $dir, $KEYNAME;
        out( 
            `gpg --yes -abs -u $KEYNAME -o $dir/Release.gpg $dir/Release`
        );
    }



    #-- perform all index generation things at once
    function Generate() {
        global $dir, $argv0;
        out(`$argv0 Packages $dir > $dir/Packages`);
        out(`$argv0 Packages.bz2 $dir`);
        out(`$argv0 Release $dir > $dir/Release`);
        out(`$argv0 Release.gpg $dir`);
    }


}//main






#-- utility code --




/**
 * Fetch known commandline arguments.
 *
 */
class argv {

    // standalone -opt
    function opt() {
        $args = func_get_args();
        return count(array_intersect($_SERVER["argv"], $args));
    }

    // parameter --arg VALUE
    function param($spec) {
        $spec = func_get_args();
        foreach ($_SERVER["argv"] as $i=>$arg) {
            if (in_array($arg, $spec)) return current(array_splice($_SERVER["argv"], $i+1, 1));
        }
    }

    // anything that doesn't look like a parameter
    function files() {
        return preg_grep('/^[^-]/', array_slice($_SERVER["argv"], 1));
    }

}//argv


/**
 * Convert list of dirs into filename list.
 *
 * @return iterator
 */
function files($dir) {
    return new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir[0]));
}



/**
 * Write list of strings to stdout, or any open file handle.
 *
 */
function out(/*...*/) {
    global $output_fh;
    $args = func_get_args();
    fprintf($output_fh, join($args));
}



#-- repository generation --



/**
 * Rewrite or normalize repository entry.
 *
 */
class/*group*/ rep_struct {


    /**
     * Replaces empty lines (or just with spaces) with a " ." placeholder.
     *
     */
    function fix_emptylines($str) {
        return preg_replace(RX_EMPTY_LINES, " .\n", $str);
    }


    /**
     * Fix blank lines in Description: field, also cut it down to 3
     * paragraphs.
     *
     */
    function description($help) {
        if (preg_match(RX_3_PARAGRAPHS, trim($help), $m)) {
            $help = $m[0];   // cut down to 3 paragraphs
        }
        $help = preg_replace("/^/m", " ", $help);  // prepend spaces on each line
        return rep_struct::fix_emptylines($help);
    }


    /**
     * Join rfc821-style Key:Value list into a text block.
     *
     */
    function join_key_value_list($info) {
        $out = "";
        foreach ($info as $key=>$value) {
            $out .= "$key: " . trim($value) . "\n";
        }
        return $out;
    }


    /**
     * Extracts valid Lines: from a control block.
     *
     */
    function entry($control) {
        preg_match_all(RX_RFC821_SPLIT, $control, $m);
        return array_combine($m[1], $m[2]);
    }
    
    /**
     * Reassembles control block, reorders entries.
     *
     */
    function normalize($control) {

        // pre-fix, split
        $control = rep_struct::fix_emptylines($control);
        $entry = rep_struct::entry($control);

        // move Description: entry
        foreach (array("Description") as $key) {
            if (isset($entry[$key])) {
                $tmp = $entry[$key];
                unset($entry[$key]);
                $entry[$key] = $tmp;
            }
        }

        // merge lines again
        return rep_struct::join_key_value_list($entry);
    }

}//rep_struct



#-- plugin meta data


/**
 * Transform package types.
 *
 */
class/*group*/ convert {


    /**
     * Extract PHP plugin meta block from filename.
     *
     */
    function php($src, $basename) {
    
        // parse /*key:value*/ comment block
        $p = new generic_pmd();
        $info = @( $p->parse($src) );
        $info["id"] = strtok(basename($basename), "-.");
        $info["architecture"] = "all";
        
        // convert and assemble into debian control block
        if (convert::is_packageable_plugin($info)) {
            $info = convert::rewrite_info($info);
            return $out = rep_struct::join_key_value_list($info);
        }
    }


    /**
     * Filter for which packages are considered real packages.
     *
     */
    function is_packageable_plugin($info) {
        return @(
            TRUE
            && $info["version"]   // must have at least version:, id: and title: fields
            && $info["id"]
            && $info["title"]
            && !in_array($info["type"], array("R", "ignore"))  // ignore faux plugin types
            && !in_array($info["api"], array(""))   // ignore api: types
            && $info["package"] != "false"
        );
    } 


    /**
     * Transpose PHP plugin standard field: names onto Debian: control keys.
     *
     */
    function rewrite_info($info) {
        $out = array();
        $map = array(
            "id" => "Package",
            "author" => "Maintainer",
            "category" => "Section",
        );
        
        #-- merge title: and help: into Description:, but keep separate Title:
        $info["description"] = @trim($info["description"] . "\n" . rep_struct::description($info["help"]));
        unset($info["help"]);
        
        #-- map
        foreach ($info as $key=>$value) {
            if (isset($map[$key])) {
                $key = $map[$key];
            }
            $key = ucwords($key);
            $value = preg_replace(RX_INDENT_RAW_TEXT, " ", $value);   // indent raw text lines
            $out[$key] = $value;
        }
        return $out;
    }


    /**
     * Get entry from zip file, whose name matches "basename.php"
     *
     */
    function zip($fn) {
        $base = strtok(basename($fn), "-.");
        $z = new ZipArchive;
        $z->open($fn);
        if ($i = $z->locateName("$base.php")) {
            return $z->getFromIndex($i);
        }
        else {
            return $z->getFromIndex(0);
        }
    }


    /**
     * Convert RPM print output into field list. Rewrite some entries.
     *
     */
    function rpm($text) {

        // split
        list($header, $body) = explode("\n\n", $text, 2);
        preg_match_all(RX_RPM_QIP_HEADER, $header, $match);
        $info = array_combine($match[1], $match[2]);

        // rewrite fields
        $map = array(
            "Name" => "Package",
            "Packager" => "Maintainer",
            "Group" => "Section",
            "Vendor" => "Author",
            "Summary" => "Title",
        );
        foreach ($info as $key=>$value) {
            $key = strtr($key, " ", "-");
            $key = isset($map[$key]) ? $map[$key] : $key;
            $control[$key] = $value;
        }
        $control["Description"] = rep_struct::description(trim($body));

        // assemble
        return
        $out = rep_struct::join_key_value_list($control);
    }


    /**
     * Read in binary file, check at end for a block of Key:value lines.
     *
     */
    function binary_extract_rfc821($fn) {
        if ($binary = file_get_contents($fn)) {
            if (preg_match(RX_RFC821_BLOCK, $binary, $m)) {
                $text = $m[0];
                $text = preg_replace("/\R/", "\n", $text);
                $text = "Architecture: win32\n" . $text;
                return $text;
            }
        }
    }

}//convert



#-- cache --

/**
 *  Stores control block entries in a cache file.
 *
 */
class cache extends PDO {

    function __construct($fn=":memory:") {
        parent::__construct("sqlite:$fn");
        $this->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_WARNING);
        $this->query("CREATE TABLE IF NOT EXISTS cache (fn VARCHAR PRIMARY KEY, time INT, control TEXT)");
    }
    
    function fetch($fn) {
        $s = $this->prepare("SELECT control FROM cache WHERE fn=? AND time=?")
         and $r = $s->execute(array($fn, filemtime($fn)))
         and $o = $s->fetch();
         return $o ? $o["control"] : NULL;
    }

    function store($fn, $data) {
        $this->prepare("INSERT OR REPLACE INTO cache (fn, time, control) VALUES (?,?,?)")
             ->execute(array($fn, filemtime($fn), $data));
        return $data;
    }
}
class nocache {   // fallback just in case if PDO or sqlite interface are missing
    function fetch($fn) { return NULL; }
    function store($fn, $data) { return $data; }
}


// if you are super concerned about closetags, use `phptags --unclosed` (or simpler `--whitespace` for fixing the actual issues, not the umbrage)
?>