Cross package maker. DEB/RPM generation or conversion. Derived from jordansissel/fpm.

βŒˆβŒ‹ βŽ‡ branch:  cross package maker


Artifact [35d5f731ae]

Artifact 35d5f731ae94db88356bd05042812970781261d0:

  • File lib/fpm/package/src.rb — part of check-in [e6f0cd3d11] at 2015-04-14 01:02:39 on branch trunk — Guard absent meta["comment"]. (user: mario size: 10239)

# encoding: ascii
# api: fpm
# title: generic script source
# description: Utilizes `pack:` specifier from meta/comment block in script files
# type: package
# category: source
# version: 0.8.3
# state: beta
# architecture: all
# license: MITL
# author: mario#include-once:org
# config: <arg name="src-only" description="Only apply main pack: spec. Don't scan recursively.">
# depends:
# pack: src.rb, README=README.txt, src/*.png
# 
# The "src" input is intended for packaging scripting language files.
# A top-level comment can hold per-file description fields, where the
# `pack:` lines simply reference other files to recurse and include.
#
#  β†’ Other documentation/meta fields from the origin file are used as
#    package attributes (fpm --flag values still take precedence).
#
#  β†’ The pack: line gives a simple comma-separated list of other scripts
#    or files to include.
#
#    Β· For instance `pack: src.rb, readme.txt, install.sh` will package
#      those files together with the main script.
#
#    Β· RECURSION: referenced files can itself specify a `pack:` line
#      onto other scripts/source/binary files.
#
#    Β· With `pack: m.py=main.py, b=b.txt` files can be renamed.
#
#    Β· Glob matching is also possible `pack: plugins/*.php`
#      Limited glob-rewriting via `doc/*.txt=manual/` even.
#      It's sort of working, use with care.
#
#    Β· A file can even exclude itself with `pack: empty.rb=`
#
#    Binary files can be referenced from a text/source file of course.
#    
#    Alternatively a concise spec.txt can be crafted to hold default
#    packaging settings for fpm. Which is a nice alternative to a "dir"
#    source or --inputs list for small projects.
#
#  β†’ `Depends:` lines aren't used for packaging (yet). They're neither
#    transcribed into system package (rpm/deb) control fields.
#    (They're mostly intended for in-application plugin management.)
#

require "fpm/package"
require "fpm/util"
require "backports"
require "fileutils"
require "find"
require "socket"
require "pathname"
require "pp"

# Source package.
#
# Reads in the meta data comment block from the first specified file.
# Recursively includes all pack:-mentioned scripts, while honoring
# src=dest maps, directory prefixes, and file glob patterns.
#
class FPM::Package::Src < FPM::Package  # inheriting from ::Dir doesn't work
  
  option "--depends", :flag, "Use depends: field as system package dependencies.", :default => false
  option "--only", :flag, "Only apply origin files pack: directive, do not recurse.", :default => false

  
  # Start from first specified source file.
  def input(path)
  
    # The init/main file should contain usable package meta fields
    if m = get_meta(path)
      # copy attributes over, if not present/overriden
      @name = m["id"] if not @name
      @version = m["version"] if not @version
      @epoch = m["epoch"] if not @epoch
      @architecture = m["architecture"] if not attributes[:architecture_given?]
      @description = "#{m['description']}\n#{(m['comment'] || "").strip}" if not attributes[:description_given?]
      @url = m["url"] || m["homepage"] if not attributes[:url_given?]
      @category = m["category"] if not attributes[:category_given?]
      @priority = m["priority"] if not attributes[:priority_given?]
      @license = m["license"] if not attributes[:license_given?]
      @vendor = m["author"] if not attributes[:maintainer_given?]
      @maintainer = m["author"] if not attributes[:maintainer_given?]
      # retain all script meta fields
      @attrs.merge!(m)
      # Keep input filename as `main` reference (for phar target)
      @attrs["main"] ||= path
      #@todo preparse and collect config: structures
    end

    # Assemble rewrite map
    # {
    #   :spec_start => { infile=>infile }
    #   rel/src1 => { src2=>dest, src3=>dest } ,
    #   rel/src2 => { src4=> .. }
    # }
    path = consolidate(path)  # strip ./ prefix if any
    @map = {:spec_start => {path => path}}
    ::Dir.chdir(attributes[:chdir] || ".") do
      rewrite_map(path)
      logger.warn(@map.pretty_inspect)
      # And copie files to staging path.
      copy_by_map({})
    end
  end


  protected

  
  # Copy all referenced files.
  #
  #
  #
  def copy_by_map(copied)
    @map.each do |from,pack|
      pack.each do |src,dest|

        # Each file can hold a primary declaration and override its target filename / make itself absent.
        if dest and @map[dest] and @map[dest][dest]
          dest = @map[dest][dest]
        end
        # References above the src/.. would also cross the staging_dir, give warning
        if dest and dest.match(/^\.\.\//)
          logger.warn("Referencing above ../ the basedir from #{from}")
        end

        # Else just apply pack{} map copying verbatim, possibly overwriting or duplicating files.
        if not copied[src+">"+dest]
          # For absolute dest=/targets: omit the --prefix path
          if attributes[:prefix] and (Pathname.new dest).relative?
            dest = attributes[:prefix] + "/" + dest
          end

          real_copy("#{src}", "#{staging_path}/#{dest}") if dest != ""
          copied[src+">"+dest] = 1
        end
      end
    end
  end

  
  # Scan files for file,src=dest mapping lists.
  #
  # Each source file can specify references as `pack: two.sh, sub/three.py`
  # where each scriptsΒ΄ subdirectory becomes the next base directory.
  #
  # TODO: The `destp` parameter is carried over from the patent old=new
  # filename pack{} map, should be joined in (doesn't work yet).
  #
  def rewrite_map(from, dest_p=nil)

    # lazy workaround to prevent neverending loops/references
    return if @map[from] and @map[from][from]
    # check if file is present, get meta data
    if not File.exists?(from)
      logger.warn("Skipping non-existent #{from} file in #{::Dir.pwd}")
      return
    elsif File.directory?(from)
      @map[from] = {}
      return
    else 
      packlist = get_meta(from)["pack"]  # adding [splitfn(from)[1]] itself would override renames
    end
    
    # relative dir and basename
    dir, fn = splitfn(from)   # fpm/ src.rb

logger.warn("from=#{from} - dest_p=#{dest_p} - dir=#{dir} - fn=#{fn} - #pack= #{packlist}")

    # @example
    #   from = fpm/src.rb
    #          `pack: exe.rb, ../README, foo=bar, test/*`
    # @result
    #   pack[fpm/src.rb] = fpm/exe.rb=>fpm/exe.rb, README=>README, fpm/foo=>fpm/bar, fpm/test/123=>...}

    # iterate over listed references
    @map[from] = pack = {}
    packlist.each do |fnspec|
    
      # specifier can be one of: `src, src=dest, src*, dir/src*=dest/`
      src, dest = fnspec.split("=", 2) # dest can be nil (=use src basename), or empty "" (=skip file)

      # join eventual parent dest_p (module/two/) and new dest (sub/)
      if dest_p
        if dest
          if (Pathname.new dest).relative?
            dest = consolidate(dest_p + dest)
            p "--- (merge dest #{dest} and #{dest_p}, ignore srcdir #{dir})"
          end
        else
          dest ||= dest_p
        end
      end
logger.warn("+ #{src}=#{dest}")

      # glob expansion
      if src.match(/[\[\{\*\?]/)
        src = ::Dir.glob(dir + src)
        logger.warn("Nothing matched for '#{fnspec}' in '#{from}'") if src.empty?
      else
        src = [dir + src]
      end

      # combine with dest, and recursively scan each referenced source file
      src.each do |src|
        if dest_p
          pack[consolidate(src)] = consolidate(dest)  # Eeek, needs dir prefix merged somehow
        else
          pack[consolidate(src)] = consolidate(dest ? dir+dest : src)
        end
        rewrite_map(consolidate(src), dironly(dest)) if not @attributes[:src_only?]
      end

    end
  end # def rewrite_map


  # split dir/name/ from basename 
  def splitfn(path)
    path =~ /\A  ((?: .*\/ )?)  ([^\/]+)  \Z/x
    return [$1, $2]
  end

  # get only dir/name/ until trailing slash from path (filename is optional)
  def dironly(path)
    path =~ /\A  ((?: .*\/ )?)  ([^\/]*)  \Z/x
    return $1
  end

  
  # remove ./ and dir/../ segments
  def consolidate(path)
    path = path.gsub(/(?<=^|\/)\.\//, "")    # strip out "./" self-referenting dirs
    path = path.gsub(/(?<=^|\/)(?!\.\.\/)[\w.-]+\/\.\.\//, "")    # consolidate "sub/../" relative paths
  end

  
  # Extract meta: key/values from source file
  def get_meta(path)

    # read file (first 8K would suffice)
    src = File.read(path, 1<<13, 0, :encoding=>'ASCII-8BIT')
    src = brute_force_charset(src)
    fields = { "pack" => "" }

    # extract first comment block,
    if src and src =~ /( \/\*+ .+? \*\/ | (?:[ \t]* \* | \#(?!!) [^\n]*\n)+ )/mix
      src = $1.gsub(/^[ \t]*(\*+|\/+|\#+)[ \t]*/, "").to_s # does not honor indentation

      # split meta block from comment (on first empty line)
      src, fields["comment"] = src.split(/\r?\n\r?\n/, 2)  # eh, Ruby, Y U NO PROVIDE \R ?
      # split key: value fields
      fields.merge!(Hash[
         src.to_s.scan(/^([\w-]+): [[:blank:]]* ([^\n]* (?:\n (?![\w-]+:) [^\n]+)* )/x).map{ |k,v| [k.downcase, v.strip] }
      ])

      # use input basename or id: field as package name
      fields["id"] = fields["id"] || path.match(/([^\/]+?)(\.\w+)?\Z/)[1]
      # split up pack: comma-separated list, don't expand src=dest yet
#p fields
      fields["pack"] = fields["pack"].split(/(?<!\\) (?!\s*=) [,\s]+ /x).map{|x|x.strip}.compact.reject{|x|x.empty?}
    else 
      fields["pack"] = []
    end
    return fields
  end # def meta
  
  def brute_force_charset(str)
    if not str
      str = ""
    end
    str.force_encoding("UTF-8")
    if !str.valid_encoding?
      str.force_encoding("ISO-8859-1")
      if !str.valid_encoding?
        return str.encode("ASCII", :invalid=>:replace, :undef=>:replace)
      end
    end
    return str
  end


  # automatically recurses for and creates subdirectories when copying
  def real_copy(src, dest)
    if dest[-1] == "/"  # trailing slash indicates target dir
      dest += splitfn(src)[1]   # therefore copy src basename
    end
    if File.exists?(src)
      FileUtils.mkdir_p(File.dirname(dest))
      FileUtils.cp_r(src, dest)
    else
      logger.info("'#{src}' still missing, eh?")
    end
  end

end # class FPM::Package::Src