⌈⌋ branch:  freshcode


Artifact Content

Artifact 99d46aea42969a9274f3d4d561c9ad2b62fa98d2:

  • File cron.daily/poll_launchpad.php — part of check-in [d16f5b9864] at 2015-01-18 22:31:51 on branch trunk — Introduce `launchpad-releases` page. Updated polling with less delay. (So far only 30% of projects information received, less than 12% of release infos.) (user: mario size: 10784)

<?php
/**
 * title: Launchpad projects/series/releases polling
 * description: Poll Launchpad devel API for project names, then releases and series
 * version: 0.2
 * category: rpc
 * api: cli
 * depends: config.local
 * type: cron
 * x-cron: *\/3 9 * * *
 * doc: https://launchpad.net/+apidoc/devel.html#projects
 *
 *
 * Retrieves project summaries from launchpad. In steps of 100.
 * Continously queries and updates release/series lists. Delays of 20 secs each.
 * Script is run in 3-minute intervals.
 * Project list should be complete in 10 days.
 * Release queries ought to be completed every 4 weeks.
 *
 *
 * # projects
 *     →  https://api.launchpad.net/devel/projects
 *     =  {
 *           "resource_type_link" : "https://api.launchpad.net/devel/#projects",
 *           "total_size" : 35449,
 *           "next_collection_link" : "https://api.launchpad.net/devel/projects?ws.size=75&memo=75&ws.start=75",
 *           "entries" : [
 *              {
 *                 "reviewer_whiteboard" : "tag:launchpad.net:2008:redacted",
 *                 "license_info" : "Launchpad 30-day trial commercial license",
 *                 "web_link" : "https://launchpad.net/breakmywork",
 *                 "official_bug_tags" : [],
 *                 "translationgroup_link" : null,
 *                 "bug_reported_acknowledgement" : "I will set a Priority AND/OR Update the bug status and take appropriate action.",
 *                 "icon_link" : "https://api.launchpad.net/devel/breakmywork/icon",
 *                 "qualifies_for_free_hosting" : true,
 *                 "registrant_link" : "https://api.launchpad.net/devel/~ravikiran-j",
 *                 "branch_sharing_policy" : "Forbidden",
 *                 "logo_link" : "https://api.launchpad.net/devel/breakmywork/logo",
 *                 "name" : "breakmywork",
 *                 "information_type" : "Public",
 *                 "licenses" : [
 *                    "Simplified BSD Licence"
 *                 ],
 *                 "commercial_subscription_is_due" : false,
 *                 "active_milestones_collection_link" : "https://api.launchpad.net/devel/breakmywork/active_milestones",
 *                 "translations_usage" : "Unknown",
 *                 "programming_language" : "python",
 *                 "freshmeat_project" : null,
 *                 "project_reviewed" : "tag:launchpad.net:2008:redacted",
 *                 "homepage_url" : null,
 *                 "development_focus_link" : "https://api.launchpad.net/devel/breakmywork/trunk",
 *                 "display_name" : "BreakMyWork",
 *                 "translationpermission" : "Open",
 *                 "bug_tracker_link" : null,
 *                 "commercial_subscription_link" : null,
 *                 "download_url" : "https://launchpad.net/~ravikiran-j/+archive/break-my-work",
 *                 "is_permitted" : "tag:launchpad.net:2008:redacted",
 *                 "all_specifications_collection_link" : "https://api.launchpad.net/devel/breakmywork/all_specifications",
 *                 "bug_reporting_guidelines" : "Please fill the following details while filing a bug:-\n1) OS Name\n2) OS Version\n3) Bug Severity (S1 - Critical, S2- High, S3 - Medium, S4 - Low)\n4) Bug Title\n5) Bug Description\n6) Bug Reproduction Steps",
 *                 "license_approved" : "tag:launchpad.net:2008:redacted",
 *                 "bug_sharing_policy" : "Public",
 *                 "brand_link" : "https://api.launchpad.net/devel/breakmywork/brand",
 *                 "self_link" : "https://api.launchpad.net/devel/breakmywork",
 *                 "releases_collection_link" : "https://api.launchpad.net/devel/breakmywork/releases",
 *                 "summary" : "RSI Prevention Software which enables the user to take timely breaks and suggests exercises/stretches.",
 *                 "date_created" : "2013-04-10T00:15:17.366157+00:00",
 *                 "owner_link" : "https://api.launchpad.net/devel/~ravikiran-j",
 *                 "active" : true,
 *                 "bug_supervisor_link" : "https://api.launchpad.net/devel/~ravikiran-j",
 *                 "remote_product" : null,
 *                 "valid_specifications_collection_link" : "https://api.launchpad.net/devel/breakmywork/valid_specifications",
 *                 "project_group_link" : null,
 *                 "wiki_url" : null,
 *                 "private" : false,
 *                 "http_etag" : "\"9f404e137b3e5c49cd16def94113c630634712de-71f9e4acddb01d68c63c3197277bf24dc096bbe7\"",
 *                 "sourceforge_project" : null,
 *                 "series_collection_link" : "https://api.launchpad.net/devel/breakmywork/series",
 *                 "translation_focus_link" : null,
 *                 "resource_type_link" : "https://api.launchpad.net/devel/#project",
 *                 "driver_link" : "https://api.launchpad.net/devel/~ravikiran-j",
 *                 "screenshots_url" : "http://imgur.com/a/MNfTw",
 *                 "specification_sharing_policy" : "Proprietary",
 *                 "recipes_collection_link" : "https://api.launchpad.net/devel/breakmywork/recipes",
 *                 "title" : "RSI Prevention Software",
 *                 "all_milestones_collection_link" : "https://api.launchpad.net/devel/breakmywork/all_milestones",
 *                 "description" : null
 *              },
 *          ],
 *           "start" : 0
 *        }
 *        
 * # releases
 *     → https://api.launchpad.net/devel/domore/releases
 *     =  { "start" : 0, "entries" : [
 *              { "changelog" : null, "project_link" :
 *                 "https://api.launchpad.net/devel/domore", "milestone_link" :
 *                 "https://api.launchpad.net/devel/domore/+milestone/1.0",
 *                 "resource_type_link" :
 *                 "https://api.launchpad.net/devel/#project_release", "http_etag" :
 *                 "\"70c989a5e150c9277c69384c6f7ba56f2e311c46-c6f49a8f8a966681af201e4654971a1d2b1ef92c\"",
 *                 "date_created" : "2012-12-09T15:06:24.291661+00:00", "display_name"
 *                 : "DoMore 1.0", "release_notes" : null, "title" : "DoMore 1.0",
 *                 "files_collection_link" :
 *                 "https://api.launchpad.net/devel/domore/trunk/1.0/files",
 *                 "self_link" : "https://api.launchpad.net/devel/domore/trunk/1.0",
 *                 "owner_link" : "https://api.launchpad.net/devel/~cseslam",
 *                 "web_link" : "https://launchpad.net/domore/trunk/1.0",
 *                 "date_released" : "2012-12-09T17:05:00+00:00", "version" : "1.0" }
 *                 ], "resource_type_link" :
 *           "https://api.launchpad.net/devel/#project_release-page-resource",
 *           "total_size" : 1
 *        }
 *
 *
 * # local cache database
 *
 *  CREATE TABLE projects ( 
 *      name                     VARCHAR  PRIMARY KEY  UNIQUE,
 *      json                     BLOB,
 *      web_link,
 *      programming_language,
 *      homepage_url,
 *      display_name             TEXT,
 *      summary                  TEXT,
 *      active                   BOOLEAN,
 *      sourceforge_project,
 *      title                    TEXT,
 *      description              TEXT,
 *      screenshots_url,
 *      releases_collection_link,
 *      download_url 
 *  );
 *  CREATE TABLE poll ( 
 *      start      INT,
 *      total_size INT 
 *  );
 *
 *
 *
 */


// Common settings
chdir(dirname(__DIR__));
include("./shared.phar");
include_once("lib/db.php");
define("FRESHCODE_USER_AGENT", "freshcode/0.7.9 (Linux x86-64; PHP/5.6.4) launchpad-poll/0.2 +http://fossil.include-once.org/freshcode/wiki/Autoupdate +mario@freshcode.club");
curl::$defaults["useragent"] = FRESHCODE_USER_AGENT;

// Separate github.releases database
db(new PDO("sqlite:launchpad.db"));




#-- complete projects list

// Start from where we left off
$start = db("SELECT start FROM poll")->start;

// complete project summary list
if ($start < 35000) {
   $projects = lp::projects($start);
   array_map("lp::insert_proj", $projects);
}



#-- now step-through poll for releases
$names = db("SELECT name FROM projects ORDER BY RANDOM() LIMIT 55")->fetchAll();
$t_outdated = time() - 21*24*3600;
print_r($names);
foreach ($names as $row) {
    $name = $row["name"];
    
    // check if it exists, or entries too old
    if ($t_outdated > db("SELECT t FROM releases WHERE name=? ORDER BY t DESC", $name)->t) {
    
        // collect
        if (count($entries = lp::proj_releases($name))) {
            $entries = array_slice($entries, -3);
        }
        else {  // stub entry to keep name+timestamp
            $entries = [["name"=>$name]];
        }

        // clean up old entries
        db("DELETE FROM releases WHERE name=?", $name);

        // and insert
        foreach ($entries as $ver) {
            $ver["name"] = $name;
            lp::insert_release($ver);
            print_r($ver);
        }
    }
    
    // delay
    sleep(9);
}





// API queries
class lp {


    // fetch proj list
    function projects($start) {
        $json = curl("https://api.launchpad.net/devel/projects?ws.size=100&memo=$start&ws.start=$start")->exec();
        if ($json and $data = json_decode($json, TRUE)) {
        
            // update poll db
            db("UPDATE poll SET start=?, total_size=?", $data["start"] + 100, $data["total_size"]);
            
            // return just contents
            return $data["entries"];
        }
        return [];
    }
    
    // store project data
    function insert_proj($data) {
        $proj_fields = "name,web_link,programming_language,homepage_url,display_name,summary,active,sourceforge_project,title,description,screenshots_url,releases_collection_link,download_url";
        $data = self::compact($data, $proj_fields);
        db("INSERT INTO projects (:?) VALUES (::)", $data, $data);
    }


    // fetch releases    
    function proj_releases($name) {
        $json = curl("https://api.launchpad.net/devel/$name/releases")->exec();
        if ($json and $data = json_decode($json, TRUE)) {
            return $data["entries"];
        }
        return [];
    }

    // store project data
    function insert_release($data) {
        $fields = "name,t,json,changelog,date_created,display_name,title,release_notes,web_link,version";
        $data = self::compact($data, $fields);
        $data["t"] = time();
        db("INSERT INTO releases (:?) VALUES (::)", $data, $data);
    }
    

    
    // retain only primary DB fields, rest goes into `json`
    function compact($data, $fields) {
        // compact into allowed fields
        $r = array_intersect_key($data, array_flip(str_getcsv($fields)));
        $r["json"] = json_encode($data);
        return $r;
    }

}

?>