Skip to content

G
go
Commit e9f82e6b authored by Andrew Gerrand's avatar Andrew Gerrand
Browse files
Options

misc/dashboard: remove old python package dashboard 

This leaves only the project page, which now resides at the web root.

R=golang-dev, bsiegert, rsc
CC=golang-dev
https://golang.org/cl/5833044
parent 2ed7087c
Hide whitespace changes
Inline Side-by-side
Showing with 168 additions and 178 deletions
misc/dashboard/godashboard/_multiprocessing.py deleted 100644 → 0
View file @ 2ed7087c
# Copyright 2009 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
import multiprocessing
misc/dashboard/godashboard/app.yaml
View file @ e9f82e6b
...@@ -11,16 +11,5 @@ handlers: ...@@ -11,16 +11,5 @@ handlers:
- url: /static - url: /static
static_dir: static static_dir: static
- url: /package - url: /(|project(|/login|/edit))
script: package.py script: project.py
- url: /package/daily
script: package.py
login: admin
- url: /project.*
script: package.py
- url: /
static_files: main.html
upload: main.html
misc/dashboard/godashboard/auth.py deleted 100644 → 0
View file @ 2ed7087c
# Copyright 2011 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
import hmac
# local imports
import key
def auth(req):
k = req.get('key')
return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey
misc/dashboard/godashboard/const.py
View file @ e9f82e6b
...@@ -3,11 +3,5 @@ ...@@ -3,11 +3,5 @@
# license that can be found in the LICENSE file. # license that can be found in the LICENSE file.
mail_from = "Go Dashboard <builder@golang.org>" mail_from = "Go Dashboard <builder@golang.org>"
mail_submit_to = "adg@golang.org" mail_submit_to = "adg@golang.org"
mail_submit_subject = "New Project Submitted" mail_submit_subject = "New Project Submitted"
mail_fail_to = "golang-dev@googlegroups.com"
mail_fail_reply_to = "golang-dev@googlegroups.com"
mail_fail_subject = "%s broken by %s"
misc/dashboard/godashboard/cron.yaml deleted 100644 → 0
View file @ 2ed7087c
cron:
- description: daily package maintenance
url: /package/daily
schedule: every 24 hours
misc/dashboard/godashboard/fail-notify.txt deleted 100644 → 0
View file @ 2ed7087c
Change {{node}} broke the {{builder}} build:
http://godashboard.appspot.com/log/{{loghash}}
{{desc}}
http://code.google.com/p/go/source/detail?r={{node}}
$ tail -n 100 < log
{{log}}
misc/dashboard/godashboard/key.py.dummy deleted 100644 → 0
View file @ 2ed7087c
# Copyright 2009 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
# Copy this file to key.py after substituting the real key.
# accessKey controls private access to the build server (i.e. to record new
# builds). It's tranmitted in the clear but, given the low value of the target,
# this should be sufficient.
accessKey = "this is not the real key"
misc/dashboard/godashboard/main.html deleted 100644 → 0
View file @ 2ed7087c
<!DOCTYPE HTML>
<html>
<head>
<title>Build Status - Go Dashboard</title>
<link rel="stylesheet" type="text/css" href="static/style.css">
</head>
<body>
<ul class="menu">
<li>Build Status</li>
<li><a href="/package">Packages</a></li>
<li><a href="/project">Projects</a></li>
<li><a href="http://golang.org/">golang.org</a></li>
</ul>
<h1>Go Dashboard</h1>
<h2>Build Status</h2>
<p class="notice">The build status dashboard has moved to <a href="http://build.golang.org">build.golang.org</a>.</p>
</body>
</html>
misc/dashboard/godashboard/package.html deleted 100644 → 0
View file @ 2ed7087c
<!DOCTYPE HTML>
<html>
<head>
<title>Packages - Go Dashboard</title>
<link rel="stylesheet" type="text/css" href="static/style.css">
</head>
<body>
<ul class="menu">
<li><a href="/">Build Status</a></li>
<li>Packages</li>
<li><a href="/project">Projects</a></li>
<li><a href="http://golang.org/">golang.org</a></li>
</ul>
<h1>Go Dashboard</h1>
<p>
Packages listed on this page are written by third parties and
may or may not build or be safe to use.
</p>
<!--
<p>
An "ok" in the <b>build</b> column indicates that the package is
<a href="http://golang.org/cmd/goinstall/">goinstallable</a>
with the latest
<a href="http://golang.org/doc/devel/release.html">release</a> of Go.
</p>
-->
<p>
The <b>info</b> column shows the first paragraph from the
<a href="http://golang.org/doc/articles/godoc_documenting_go_code.html">package doc comment</a>.
</p>
<h2>Most Installed Packages (this week)</h2>
<table class="alternate" cellpadding="0" cellspacing="0">
<tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr>
{% for r in by_week_count %}
<tr>
<td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td>
<td class="count">{{r.week_count}}</td>
<!-- <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %}&nbsp;{% endif %}</td> -->
<td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td>
<td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td>
</tr>
{% endfor %}
</table>
<h2>Recently Installed Packages</h2>
<table class="alternate" cellpadding="0" cellspacing="0">
<tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr>
{% for r in by_time %}
<tr>
<td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td>
<td class="count">{{r.count}}</td>
<!-- <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %}&nbsp;{% endif %}</td> -->
<td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td>
<td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td>
</tr>
{% endfor %}
</table>
<h2>Most Installed Packages (all time)</h2>
<table class="alternate" cellpadding="0" cellspacing="0">
<tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr>
{% for r in by_count %}
<tr>
<td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td>
<td class="count">{{r.count}}</td>
<!-- <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %}&nbsp;{% endif %}</td> -->
<td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td>
<td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td>
</tr>
{% endfor %}
</table>
</body>
</html>
misc/dashboard/godashboard/project-edit.html
View file @ e9f82e6b
...@@ -19,8 +19,6 @@ Tags: (comma-separated)<br/> ...@@ -19,8 +19,6 @@ Tags: (comma-separated)<br/>
<input type="text" id="tags" name="tags" value="{{tags}}"><br/> <input type="text" id="tags" name="tags" value="{{tags}}"><br/>
Web URL:<br/> Web URL:<br/>
<input type="text" name="web_url" value="{{p.web_url|escape}}"><br/> <input type="text" name="web_url" value="{{p.web_url|escape}}"><br/>
Package URL: (to link to a goinstall'd package)<br/>
<input type="text" name="package" value="{{p.package.path|escape}}"><br/>
Approved: <input type="checkbox" name="approved" value="1" {% if p.approved %}checked{% endif %}><br/> Approved: <input type="checkbox" name="approved" value="1" {% if p.approved %}checked{% endif %}><br/>
<br/> <br/>
<input type="submit" name="do" value="Save"> <input type="submit" name="do" value="Save">
......
misc/dashboard/godashboard/project.html
View file @ e9f82e6b
<!DOCTYPE HTML> <!DOCTYPE HTML>
<html> <html>
<head> <head>
<title>Projects - Go Dashboard</title> <title>Go Projects</title>
<link rel="stylesheet" type="text/css" href="static/style.css"> <link rel="stylesheet" type="text/css" href="static/style.css">
<style>
.unapproved a.name { color: red }
.tag { font-size: 0.8em; color: #666 }
</style>
</head> </head>
<body> <body>
<ul class="menu"> <h1>Go Projects</h1>
<li><a href="/">Build Status</a></li>
<li><a href="/package">Packages</a></li>
<li>Projects</li>
<li><a href="http://golang.org/">golang.org</a></li>
</ul>
<h1>Go Dashboard</h1>
<p> <p>
These are external projects and not endorsed or supported by the Go project. These are external projects and not endorsed or supported by the Go project.
...@@ -80,6 +69,5 @@ ...@@ -80,6 +69,5 @@
{% endfor %} {% endfor %}
</ul> </ul>
</body> </body>
</html> </html>
misc/dashboard/godashboard/package.py → misc/dashboard/godashboard/project.py
View file @ e9f82e6b
...@@ -2,107 +2,22 @@ ...@@ -2,107 +2,22 @@
# Use of this source code is governed by a BSD-style # Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file. # license that can be found in the LICENSE file.
# This is the server part of the package dashboard.
# It must be run by App Engine.
from google.appengine.api import mail from google.appengine.api import mail
from google.appengine.api import memcache from google.appengine.api import memcache
from google.appengine.api import taskqueue
from google.appengine.api import urlfetch
from google.appengine.api import users from google.appengine.api import users
from google.appengine.ext import db from google.appengine.ext import db
from google.appengine.ext import webapp from google.appengine.ext import webapp
from google.appengine.ext.webapp import template from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app from google.appengine.ext.webapp.util import run_wsgi_app
import datetime
import logging
import os import os
import re
import sets import sets
import urllib2
# local imports # local imports
from auth import auth
import toutf8 import toutf8
import const import const
template.register_template_library('toutf8') template.register_template_library('toutf8')
# Storage model for package info recorded on server.
class Package(db.Model):
path = db.StringProperty()
web_url = db.StringProperty() # derived from path
count = db.IntegerProperty() # grand total
week_count = db.IntegerProperty() # rolling weekly count
day_count = db.TextProperty(default='') # daily count
last_install = db.DateTimeProperty()
# data contributed by gobuilder
info = db.StringProperty()
ok = db.BooleanProperty()
last_ok = db.DateTimeProperty()
def get_day_count(self):
counts = {}
if not self.day_count:
return counts
for d in str(self.day_count).split('\n'):
date, count = d.split(' ')
counts[date] = int(count)
return counts
def set_day_count(self, count):
days = []
for day, count in count.items():
days.append('%s %d' % (day, count))
days.sort(reverse=True)
days = days[:28]
self.day_count = '\n'.join(days)
def inc(self):
count = self.get_day_count()
today = str(datetime.date.today())
count[today] = count.get(today, 0) + 1
self.set_day_count(count)
self.update_week_count(count)
self.count += 1
def update_week_count(self, count=None):
if count is None:
count = self.get_day_count()
total = 0
today = datetime.date.today()
for i in range(7):
day = str(today - datetime.timedelta(days=i))
if day in count:
total += count[day]
self.week_count = total
# PackageDaily kicks off the daily package maintenance cron job
# and serves the associated task queue.
class PackageDaily(webapp.RequestHandler):
def get(self):
# queue a task to update each package with a week_count > 0
keys = Package.all(keys_only=True).filter('week_count >', 0)
for key in keys:
taskqueue.add(url='/package/daily', params={'key': key.name()})
def post(self):
# update a single package (in a task queue)
def update(key):
p = Package.get_by_key_name(key)
if not p:
return
p.update_week_count()
p.put()
key = self.request.get('key')
if not key:
return
db.run_in_transaction(update, key)
class Project(db.Model): class Project(db.Model):
name = db.StringProperty(indexed=True) name = db.StringProperty(indexed=True)
descr = db.StringProperty() descr = db.StringProperty()
...@@ -112,204 +27,19 @@ class Project(db.Model): ...@@ -112,204 +27,19 @@ class Project(db.Model):
tags = db.ListProperty(str) tags = db.ListProperty(str)
approved = db.BooleanProperty(indexed=True) approved = db.BooleanProperty(indexed=True)
re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$')
re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$')
re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$')
re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$')
def vc_to_web(path):
if re_bitbucket.match(path):
m = re_bitbucket.match(path)
check_url = 'http://' + m.group(1) + '/?cmd=heads'
web = 'http://' + m.group(1) + '/'
elif re_github.match(path):
m = re_github_web.match(path)
check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/'
web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/'
elif re_googlecode.match(path):
m = re_googlecode.match(path)
check_url = 'http://'+path
if not m.group(2): # append / after bare '/hg' or '/git'
check_url += '/'
web = 'http://code.google.com/p/' + path[:path.index('.')]
elif re_launchpad.match(path):
check_url = web = 'https://'+path
else:
return False, False
return web, check_url
re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)')
re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?')
re_striphttp = re.compile(r'https?://(www\.)?')
def find_googlecode_vcs(path):
# Perform http request to path/hg or path/git to check if they're
# using mercurial or git. Otherwise, assume svn.
for vcs in ['git', 'hg']:
try:
response = urlfetch.fetch('http://'+path+vcs, deadline=1)
if response.status_code == 200:
return vcs
except: pass
return 'svn'
def web_to_vc(url):
url = re_striphttp.sub('', url)
m = re_bitbucket_web.match(url)
if m:
return 'bitbucket.org/'+m.group(1)+'/'+m.group(2)
m = re_github_web.match(url)
if m:
return 'github.com/'+m.group(1)+'/'+m.group(2)
m = re_googlecode_web.match(url)
if m:
path = m.group(1)+'.googlecode.com/'
vcs = find_googlecode_vcs(path)
return path + vcs
m = re_launchpad_web.match(url)
if m:
return m.group(0)
return False
MaxPathLength = 100
CacheTimeout = 3600 CacheTimeout = 3600
class PackagePage(webapp.RequestHandler):
def get(self):
if self.request.get('fmt') == 'json':
return self.json()
html = memcache.get('view-package')
if not html:
tdata = {}
q = Package.all().filter('week_count >', 0)
q.order('-week_count')
tdata['by_week_count'] = q.fetch(50)
q = Package.all()
q.order('-last_install')
tdata['by_time'] = q.fetch(20)
q = Package.all()
q.order('-count')
tdata['by_count'] = q.fetch(100)
path = os.path.join(os.path.dirname(__file__), 'package.html')
html = template.render(path, tdata)
memcache.set('view-package', html, time=CacheTimeout)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.out.write(html)
def json(self):
json = memcache.get('view-package-json')
if not json:
q = Package.all()
s = '{"packages": ['
sep = ''
for r in q:
s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count)
sep = ','
s += '\n]}\n'
json = s
memcache.set('view-package-json', json, time=CacheTimeout)
self.response.set_status(200)
self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
self.response.out.write(json)
def can_get_url(self, url):
try:
urllib2.urlopen(urllib2.Request(url))
return True
except:
return False
def is_valid_package_path(self, path):
return (re_bitbucket.match(path) or
re_googlecode.match(path) or
re_github.match(path) or
re_launchpad.match(path))
def record_pkg(self, path):
# sanity check string
if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path):
return False
# look in datastore
key = 'pkg-' + path
p = Package.get_by_key_name(key)
if p is None:
# not in datastore - verify URL before creating
web, check_url = vc_to_web(path)
if not web:
logging.error('unrecognized path: %s', path)
return False
if not self.can_get_url(check_url):
logging.error('cannot get %s', check_url)
return False
p = Package(key_name = key, path = path, count = 0, web_url = web)
if auth(self.request):
# builder updating package metadata
p.info = self.request.get('info')
p.ok = self.request.get('ok') == "true"
if p.ok:
p.last_ok = datetime.datetime.utcnow()
else:
# goinstall reporting an install
p.inc()
p.last_install = datetime.datetime.utcnow()
# update package object
p.put()
return True
def post(self):
path = self.request.get('path')
ok = db.run_in_transaction(self.record_pkg, path)
if ok:
self.response.set_status(200)
self.response.out.write('ok')
else:
logging.error('invalid path in post: %s', path)
self.response.set_status(500)
self.response.out.write('not ok')
class ProjectPage(webapp.RequestHandler): class ProjectPage(webapp.RequestHandler):
def get(self): def get(self):
admin = users.is_current_user_admin() admin = users.is_current_user_admin()
if self.request.path == "/project/login": if self.request.path == "/project/login":
self.redirect(users.create_login_url("/project")) self.redirect(users.create_login_url("/project"))
elif self.request.path == "/project/logout":
self.redirect(users.create_logout_url("/project"))
elif self.request.path == "/project/edit" and admin: elif self.request.path == "/project/edit" and admin:
self.edit() self.edit()
elif self.request.path == "/project/assoc" and admin:
self.assoc()
else: else:
self.list() self.list()
def assoc(self):
projects = Project.all()
for p in projects:
if p.package:
continue
path = web_to_vc(p.web_url)
if not path:
continue
pkg = Package.get_by_key_name("pkg-"+path)
if not pkg:
self.response.out.write('no: %s %s<br>' % (p.web_url, path))
continue
p.package = pkg
p.put()
self.response.out.write('yes: %s %s<br>' % (p.web_url, path))
def post(self): def post(self):
if self.request.path == "/project/edit": if self.request.path == "/project/edit":
self.edit(True) self.edit(True)
...@@ -419,9 +149,7 @@ class ProjectPage(webapp.RequestHandler): ...@@ -419,9 +149,7 @@ class ProjectPage(webapp.RequestHandler):
def main(): def main():
app = webapp.WSGIApplication([ app = webapp.WSGIApplication([
('/package', PackagePage), ('/.*', ProjectPage),
('/package/daily', PackageDaily),
('/project.*', ProjectPage),
], debug=True) ], debug=True)
run_wsgi_app(app) run_wsgi_app(app)
......
misc/dashboard/godashboard/static/style.css
View file @ e9f82e6b
...@@ -127,3 +127,10 @@ td.time { ...@@ -127,3 +127,10 @@ td.time {
.notice a { .notice a {
color: #FF6; color: #FF6;
} }
.unapproved a.name {
color: red;
}
.tag {
font-size: 0.8em;
color: #666;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7