<?php
/**
* title: Launchpad projects/series/releases polling
* description: Poll Launchpad devel API for project names, then releases and series
* version: 0.2
* category: rpc
* api: cli
* depends: config.local
* type: cron
* x-cron: *\/3 9 * * *
* doc: https://launchpad.net/+apidoc/devel.html#projects
*
*
* Retrieves project summaries from launchpad. In steps of 100.
* Continously queries and updates release/series lists. Delays of 20 secs each.
* Script is run in 3-minute intervals.
* Project list should be complete in 10 days.
* Release queries ought to be completed every 4 weeks.
*
*
* # projects
* → https://api.launchpad.net/devel/projects
* = {
* "resource_type_link" : "https://api.launchpad.net/devel/#projects",
* "total_size" : 35449,
* "next_collection_link" : "https://api.launchpad.net/devel/projects?ws.size=75&memo=75&ws.start=75",
* "entries" : [
* {
* "reviewer_whiteboard" : "tag:launchpad.net:2008:redacted",
* "license_info" : "Launchpad 30-day trial commercial license",
* "web_link" : "https://launchpad.net/breakmywork",
* "official_bug_tags" : [],
* "translationgroup_link" : null,
* "bug_reported_acknowledgement" : "I will set a Priority AND/OR Update the bug status and take appropriate action.",
* "icon_link" : "https://api.launchpad.net/devel/breakmywork/icon",
* "qualifies_for_free_hosting" : true,
* "registrant_link" : "https://api.launchpad.net/devel/~ravikiran-j",
* "branch_sharing_policy" : "Forbidden",
* "logo_link" : "https://api.launchpad.net/devel/breakmywork/logo",
* "name" : "breakmywork",
* "information_type" : "Public",
* "licenses" : [
* "Simplified BSD Licence"
* ],
* "commercial_subscription_is_due" : false,
* "active_milestones_collection_link" : "https://api.launchpad.net/devel/breakmywork/active_milestones",
* "translations_usage" : "Unknown",
* "programming_language" : "python",
* "freshmeat_project" : null,
* "project_reviewed" : "tag:launchpad.net:2008:redacted",
* "homepage_url" : null,
* "development_focus_link" : "https://api.launchpad.net/devel/breakmywork/trunk",
* "display_name" : "BreakMyWork",
* "translationpermission" : "Open",
* "bug_tracker_link" : null,
* "commercial_subscription_link" : null,
* "download_url" : "https://launchpad.net/~ravikiran-j/+archive/break-my-work",
* "is_permitted" : "tag:launchpad.net:2008:redacted",
* "all_specifications_collection_link" : "https://api.launchpad.net/devel/breakmywork/all_specifications",
* "bug_reporting_guidelines" : "Please fill the following details while filing a bug:-\n1) OS Name\n2) OS Version\n3) Bug Severity (S1 - Critical, S2- High, S3 - Medium, S4 - Low)\n4) Bug Title\n5) Bug Description\n6) Bug Reproduction Steps",
* "license_approved" : "tag:launchpad.net:2008:redacted",
* "bug_sharing_policy" : "Public",
* "brand_link" : "https://api.launchpad.net/devel/breakmywork/brand",
* "self_link" : "https://api.launchpad.net/devel/breakmywork",
* "releases_collection_link" : "https://api.launchpad.net/devel/breakmywork/releases",
* "summary" : "RSI Prevention Software which enables the user to take timely breaks and suggests exercises/stretches.",
* "date_created" : "2013-04-10T00:15:17.366157+00:00",
* "owner_link" : "https://api.launchpad.net/devel/~ravikiran-j",
* "active" : true,
* "bug_supervisor_link" : "https://api.launchpad.net/devel/~ravikiran-j",
* "remote_product" : null,
* "valid_specifications_collection_link" : "https://api.launchpad.net/devel/breakmywork/valid_specifications",
* "project_group_link" : null,
* "wiki_url" : null,
* "private" : false,
* "http_etag" : "\"9f404e137b3e5c49cd16def94113c630634712de-71f9e4acddb01d68c63c3197277bf24dc096bbe7\"",
* "sourceforge_project" : null,
* "series_collection_link" : "https://api.launchpad.net/devel/breakmywork/series",
* "translation_focus_link" : null,
* "resource_type_link" : "https://api.launchpad.net/devel/#project",
* "driver_link" : "https://api.launchpad.net/devel/~ravikiran-j",
* "screenshots_url" : "http://imgur.com/a/MNfTw",
* "specification_sharing_policy" : "Proprietary",
* "recipes_collection_link" : "https://api.launchpad.net/devel/breakmywork/recipes",
* "title" : "RSI Prevention Software",
* "all_milestones_collection_link" : "https://api.launchpad.net/devel/breakmywork/all_milestones",
* "description" : null
* },
* ],
* "start" : 0
* }
*
* # releases
* → https://api.launchpad.net/devel/domore/releases
* = { "start" : 0, "entries" : [
* { "changelog" : null, "project_link" :
* "https://api.launchpad.net/devel/domore", "milestone_link" :
* "https://api.launchpad.net/devel/domore/+milestone/1.0",
* "resource_type_link" :
* "https://api.launchpad.net/devel/#project_release", "http_etag" :
* "\"70c989a5e150c9277c69384c6f7ba56f2e311c46-c6f49a8f8a966681af201e4654971a1d2b1ef92c\"",
* "date_created" : "2012-12-09T15:06:24.291661+00:00", "display_name"
* : "DoMore 1.0", "release_notes" : null, "title" : "DoMore 1.0",
* "files_collection_link" :
* "https://api.launchpad.net/devel/domore/trunk/1.0/files",
* "self_link" : "https://api.launchpad.net/devel/domore/trunk/1.0",
* "owner_link" : "https://api.launchpad.net/devel/~cseslam",
* "web_link" : "https://launchpad.net/domore/trunk/1.0",
* "date_released" : "2012-12-09T17:05:00+00:00", "version" : "1.0" }
* ], "resource_type_link" :
* "https://api.launchpad.net/devel/#project_release-page-resource",
* "total_size" : 1
* }
*
*
* # local cache database
*
* CREATE TABLE projects (
* name VARCHAR PRIMARY KEY UNIQUE,
* json BLOB,
* web_link,
* programming_language,
* homepage_url,
* display_name TEXT,
* summary TEXT,
* active BOOLEAN,
* sourceforge_project,
* title TEXT,
* description TEXT,
* screenshots_url,
* releases_collection_link,
* download_url
* );
* CREATE TABLE poll (
* start INT,
* total_size INT
* );
*
*
*
*/
// Common settings
chdir(dirname(__DIR__));
include("./shared.phar");
include_once("lib/db.php");
define("FRESHCODE_USER_AGENT", "freshcode/0.7.9 (Linux x86-64; PHP/5.6.4) launchpad-poll/0.2 +http://fossil.include-once.org/freshcode/wiki/Autoupdate +mario@freshcode.club");
curl::$defaults["useragent"] = FRESHCODE_USER_AGENT;
// Separate github.releases database
db(new PDO("sqlite:launchpad.db"));
#-- complete projects list
// Start from where we left off
$start = db("SELECT start FROM poll")->start;
// complete project summary list
if ($start < 35000) {
$projects = lp::projects($start);
array_map("lp::insert_proj", $projects);
}
#-- now step-through poll for releases
$names = db("SELECT name FROM projects ORDER BY RANDOM() LIMIT 55")->fetchAll();
$t_outdated = time() - 21*24*3600;
print_r($names);
foreach ($names as $row) {
$name = $row["name"];
// check if it exists, or entries too old
if ($t_outdated > db("SELECT t FROM releases WHERE name=? ORDER BY t DESC", $name)->t) {
// collect
if (count($entries = lp::proj_releases($name))) {
$entries = array_slice($entries, -3);
}
else { // stub entry to keep name+timestamp
$entries = [["name"=>$name]];
}
// clean up old entries
db("DELETE FROM releases WHERE name=?", $name);
// and insert
foreach ($entries as $ver) {
$ver["name"] = $name;
lp::insert_release($ver);
print_r($ver);
}
}
// delay
sleep(9);
}
// API queries
class lp {
// fetch proj list
function projects($start) {
$json = curl("https://api.launchpad.net/devel/projects?ws.size=100&memo=$start&ws.start=$start")->exec();
if ($json and $data = json_decode($json, TRUE)) {
// update poll db
db("UPDATE poll SET start=?, total_size=?", $data["start"] + 100, $data["total_size"]);
// return just contents
return $data["entries"];
}
return [];
}
// store project data
function insert_proj($data) {
$proj_fields = "name,web_link,programming_language,homepage_url,display_name,summary,active,sourceforge_project,title,description,screenshots_url,releases_collection_link,download_url";
$data = self::compact($data, $proj_fields);
db("INSERT INTO projects (:?) VALUES (::)", $data, $data);
}
// fetch releases
function proj_releases($name) {
$json = curl("https://api.launchpad.net/devel/$name/releases")->exec();
if ($json and $data = json_decode($json, TRUE)) {
return $data["entries"];
}
return [];
}
// store project data
function insert_release($data) {
$fields = "name,t,json,changelog,date_created,display_name,title,release_notes,web_link,version";
$data = self::compact($data, $fields);
$data["t"] = time();
db("INSERT INTO releases (:?) VALUES (::)", $data, $data);
}
// retain only primary DB fields, rest goes into `json`
function compact($data, $fields) {
// compact into allowed fields
$r = array_intersect_key($data, array_flip(str_getcsv($fields)));
$r["json"] = json_encode($data);
return $r;
}
}
?>