Skip to content

N
nxd-bom
Commit 0daf7c4e authored by Jérome Perrin's avatar Jérome Perrin
Browse files
Options

support more download and git URLs

parent e0437e84
Show whitespace changes
Inline Side-by-side
Showing with 86 additions and 14 deletions
nxdbom/__init__.py
View file @ 0daf7c4e
...@@ -341,21 +341,40 @@ def bom_node(XXX): ...@@ -341,21 +341,40 @@ def bom_node(XXX):
# for example: # for example:
# #
# http://www.python.org/ftp/python/2.7.18/Python-2.7.18.tar.xz -> ('Python', '2.7.18') # http://www.python.org/ftp/python/2.7.18/Python-2.7.18.tar.xz -> ('Python', '2.7.18')
_gitweb_re = re.compile(r'/gitweb/\?p=(?P<name>\w+)\.git;a=snapshot;h=(?P<rev>\w+)') download_url_matchers = (
_github_re = re.compile(r'/github.com/[\w\-]+/(?P<name>[\w\-]+)/archive/(refs/tags/)?(?P<rev>.+)$') re.compile(r'/gitweb/\?p=(?P<name>\w+)\.git;a=snapshot;h=(?P<rev>\w+)'),
_github_rre= re.compile(r'/raw.githubusercontent.com/[\w\-]+/(?P<name>[\w\-]+)/(?P<rev>[\w\.\-]+)/') re.compile(r'https://github.com/libevent/(?P<name>.+)/releases/download/[^/]+/libevent-(?P<rev>[^/]+)-stable'),
_gitlab_re = re.compile(r'/lab.nexedi.com/[\w\-]+/(?P<name>[\w\-]+)/repository/archive.tar.(gz|bz2)\?ref=(?P<rev>.+)$') re.compile(r'https://github.com/(?P<name>nextcloud/.+)/releases/download/(?P<rev>[^/]+)'),
_SF_re = re.compile(r'/sourceforge.net/.+/(?P<name>[\w\_]+)-(?P<rev>[\w\.]+)/download$') re.compile(r'https://github.com/tesseract-ocr/(?P<name>tessdata)/raw/(?P<rev>.+)'),
_git_re = re.compile(r'/(?P<name>[\w\.\-]+)\.git$') re.compile(r'/github.com/[\w\-]+/(?P<name>[\w\-]+)/archive/(refs/tags/)?(?P<rev>.+)$'),
_go_re = re.compile(r'/golang.org/dl/(?P<name>go)(?P<rev>[\w\.]+).src$') re.compile(r'/raw.githubusercontent.com/[\w\-]+/(?P<name>[\w\-]+)/(?P<rev>[\w\.\-]+)/'),
_go_dev_re = re.compile(r'/go.dev/dl/(?P<name>go)(?P<rev>[\w\.]+).src$') re.compile(r'/lab.nexedi.com/[\w\-]+/(?P<name>[\w\-]+)/repository/archive.tar.(gz|bz2)\?ref=(?P<rev>.+)$'),
re.compile(r'/lab.nexedi.com/[\w\-]+/(?P<name>[\w\-]+)/repository/(?P<rev>.+)/archive'),
re.compile(r'/sourceforge.net/.+/(?P<name>[\w\_]+)-(?P<rev>[\w\.]+)/download$'),
re.compile(r'/golang.org/dl/(?P<name>go)(?P<rev>[\w\.]+).src$'),
re.compile(r'/go.dev/dl/(?P<name>go)(?P<rev>[\w\.]+).src$'),
re.compile(r'archive.debian.org/debian-archive/debian/pool/[\w\/]+/(?P<name>.+)/.*_(?P<rev>.+)(\.orig|)+$'),
re.compile(r'cdimage.debian.org/cdimage/archive/(?P<rev>.+)/amd64/iso-cd/(?P<name>[^/-]+)'),
re.compile(r'https://.*.storage.googleapis.com/(?P<rev>.+)/(?P<name>.+)-\d+(\.\d+)*'),
re.compile(r'https://.*.storage.googleapis.com/(?P<rev>.+)/(?P<name>.+)'),
re.compile(r'https://downloads.metabase.com/(?P<rev>.+)/(?P<name>.+).jar'),
re.compile(r'https://nodejs.org/download/release/v(?P<rev>.+)/(?P<name>.+)'),
re.compile(r'https://(?P<name>nodejs).org/dist/v(?P<rev>.+)/node-v'),
re.compile(r'snappymail.eu/repository/(?P<name>nextcloud/.+)-(?P<rev>.+)-nextcloud'),
re.compile(r'dlcdn.apache.org/.+/binaries/(?P<name>.+)-(?P<rev>.+)-bin'),
re.compile(r'https://inkscape.org/gallery/item/13330/(?P<name>inkscape)-(?P<rev>.+)_.'),
)
git_url_matchers = (
re.compile(r'/(?P<name>[\w\.\-]+)\.git$'),
re.compile(r'(lab\.nexedi\.com|gitlab\.com|github\.com)/[\w\.\-]+/(?P<name>[\w\.\-]+)$'),
)
_osdn_f = re.compile(r'/osdn.net/frs/redir.php\?f=(?P<f>.+)$') _osdn_f = re.compile(r'/osdn.net/frs/redir.php\?f=(?P<f>.+)$')
def namever(url, failonerr=True): # -> (name, ver) | None if !failonerr def namever(url, failonerr=True): # -> (name, ver) | None if !failonerr
# http://www.ijg.org/files/jpegsrc.v9d.tar.gz -> http://www.ijg.org/files/jpegsrc.v9d # http://www.ijg.org/files/jpegsrc.v9d.tar.gz -> http://www.ijg.org/files/jpegsrc.v9d
def del_tgztail(s): def del_tgztail(s):
for tail in ('.tgz', '.tar.gz', '.tbz', '.tar.bz2', '.tar.xz', '.tar.lz', '.zip', '.egg'): for tail in ('.tgz', '.tar.gz', '.tbz', '.tar.bz2', '.tar.xz', '.tar.lz', '.zip', '.egg'):
s = removesuffix(s, tail) s = removesuffix(s, tail)
for tail in ('-linux-x86_64', '_Linux_x86-64_rpm'): # FIXME x86/arm, non-linux ? for tail in ('-linux-x86_64', '_Linux_x86-64_rpm', '-linux64', '_linux64'): # FIXME x86/arm, non-linux ?
s = removesuffix(s, tail) s = removesuffix(s, tail)
for tail in ['-py2.7'] + ['-py3.%d' % _ for _ in range(20)]: for tail in ['-py2.7'] + ['-py3.%d' % _ for _ in range(20)]:
s = removesuffix(s, tail) s = removesuffix(s, tail)
...@@ -385,12 +404,13 @@ def namever(url, failonerr=True): # -> (name, ver) | None if !failonerr ...@@ -385,12 +404,13 @@ def namever(url, failonerr=True): # -> (name, ver) | None if !failonerr
return name, ver return name, ver
def _namever(url, failonerr): def _namever(url, failonerr):
for r in (_gitweb_re, _github_re, _github_rre, _gitlab_re, _SF_re, _go_re, _go_dev_re): for r in download_url_matchers:
m = r.search(url) m = r.search(url)
if m is not None: if m is not None:
return m.group('name'), m.group('rev') return m.group('name'), m.group('rev')
m = _git_re.search(url) for r in git_url_matchers:
m = r.search(url)
if m is not None: if m is not None:
return m.group('name'), None return m.group('name'), None
......
nxdbom/nxdbom_test.py
View file @ 0daf7c4e
...@@ -32,6 +32,7 @@ from os.path import dirname, exists ...@@ -32,6 +32,7 @@ from os.path import dirname, exists
('https://golang.org/dl/go1.18.9.src', 'go', '1.18.9'), ('https://golang.org/dl/go1.18.9.src', 'go', '1.18.9'),
('https://go.dev/dl/go1.20.6.src.tar.gz', 'go', '1.20.6'), ('https://go.dev/dl/go1.20.6.src.tar.gz', 'go', '1.20.6'),
('https://github.com/tesseract-ocr/tesseract/archive/refs/tags/4.1.1.tar.gz', 'tesseract', '4.1.1'), ('https://github.com/tesseract-ocr/tesseract/archive/refs/tags/4.1.1.tar.gz', 'tesseract', '4.1.1'),
('https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata', 'tessdata', '590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata'),
('https://raw.githubusercontent.com/zuphilip/ocropy-models/master/en-default.pyrnn.gz', 'ocropy-models', 'master'), ('https://raw.githubusercontent.com/zuphilip/ocropy-models/master/en-default.pyrnn.gz', 'ocropy-models', 'master'),
('https://sourceforge.net/projects/swig/files/swig/swig-3.0.12/swig-3.0.12.tar.gz/download', 'swig', '3.0.12'), ('https://sourceforge.net/projects/swig/files/swig/swig-3.0.12/swig-3.0.12.tar.gz/download', 'swig', '3.0.12'),
('https://git.savannah.gnu.org/gitweb/?p=config.git;a=snapshot;h=5e531d39;sf=tgz', 'config', '5e531d39'), ('https://git.savannah.gnu.org/gitweb/?p=config.git;a=snapshot;h=5e531d39;sf=tgz', 'config', '5e531d39'),
...@@ -44,11 +45,27 @@ from os.path import dirname, exists ...@@ -44,11 +45,27 @@ from os.path import dirname, exists
('http://downloadarchive.documentfoundation.org/libreoffice/old/5.2.4.2/rpm/x86_64/LibreOffice_5.2.4.2_Linux_x86-64_rpm.tar.gz', 'LibreOffice', '5.2.4.2'), ('http://downloadarchive.documentfoundation.org/libreoffice/old/5.2.4.2/rpm/x86_64/LibreOffice_5.2.4.2_Linux_x86-64_rpm.tar.gz', 'LibreOffice', '5.2.4.2'),
('http://www.cups.org/software/1.7.4/cups-1.7.4-source.tar.bz2', 'cups', '1.7.4'), ('http://www.cups.org/software/1.7.4/cups-1.7.4-source.tar.bz2', 'cups', '1.7.4'),
('https://github.com/unicode-org/icu/releases/download/release-58-2/icu4c-58_2-src.tgz', 'icu4c', '58_2'), ('https://github.com/unicode-org/icu/releases/download/release-58-2/icu4c-58_2-src.tgz', 'icu4c', '58_2'),
('https://github.com/mozilla/geckodriver/releases/download/v0.16.1/geckodriver-v0.16.1-linux64.tar.gz', 'geckodriver', '0.16.1'),
('https://github.com/libevent/libevent/releases/download/release-2.1.11-stable/libevent-2.1.11-stable.tar.gz', 'libevent', '2.1.11'),
('https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=8a40eb47bd80a40ecde14c223525b21852d2fc9f', 'onlyoffice_core', '8a40eb47bd80a40ecde14c223525b21852d2fc9f'), ('https://lab.nexedi.com/bk/onlyoffice_core/repository/archive.tar.bz2?ref=8a40eb47bd80a40ecde14c223525b21852d2fc9f', 'onlyoffice_core', '8a40eb47bd80a40ecde14c223525b21852d2fc9f'),
('http://snowball.tartarus.org/dist/libstemmer_c.tgz', 'libstemmer_c', None), ('http://snowball.tartarus.org/dist/libstemmer_c.tgz', 'libstemmer_c', None),
('https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.0-patch1/src/hdf5-1.10.0-patch1.tar.bz2', 'hdf5', '1.10.0-patch1'), ('https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.0-patch1/src/hdf5-1.10.0-patch1.tar.bz2', 'hdf5', '1.10.0-patch1'),
('http://ftp.debian.org/debian/pool/main/f/findutils/findutils_4.8.0.orig.tar.xz', 'findutils', '4.8.0'), ('http://ftp.debian.org/debian/pool/main/f/findutils/findutils_4.8.0.orig.tar.xz', 'findutils', '4.8.0'),
('strict_rfc3339-0.7-py2.7.egg', 'strict_rfc3339', '0.7'), ('strict_rfc3339-0.7-py2.7.egg', 'strict_rfc3339', '0.7'),
('http://archive.debian.org/debian-archive/debian/pool/main/m/make-dfsg/make-dfsg_3.81.orig.tar.gz', 'make-dfsg', '3.81'),
('https://archive.debian.org/debian-archive/debian/pool/main/f/fonts-ocr-b/fonts-ocr-b_0.2~dfsg1.orig.tar.gz', 'fonts-ocr-b', '0.2~dfsg1'),
('https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip', 'chromedriver', '2.41'),
('https://chromedriver.storage.googleapis.com/91.0.4472.101/chromedriver_linux64.zip', 'chromedriver', '91.0.4472.101'),
('https://selenium-release.storage.googleapis.com/3.14/selenium-server-standalone-3.14.0.jar', 'selenium-server-standalone', '3.14'),
('https://downloads.metabase.com/v0.48.2/metabase.jar', 'metabase', '0.48.2'),
('https://cdimage.debian.org/cdimage/archive/11.7.0/amd64/iso-cd/debian-11.7.0-amd64-netinst.iso', 'debian', '11.7.0'),
('https://nodejs.org/dist/v16.19.0/node-v16.19.0.tar.gz', 'nodejs', '16.19.0'),
('https://nodejs.org/download/release/v16.19.0/node-v16.19.0-headers.tar.gz', 'node-v16.19.0-headers', '16.19.0'), # XXX not perfect
('https://github.com/nextcloud/news/releases/download/24.0.0/news.tar.gz', 'nextcloud/news', '24.0.0', ),
('https://snappymail.eu/repository/nextcloud/snappymail-2.29.4-nextcloud.tar.gz', 'nextcloud/snappymail', '2.29.4'),
('https://dlcdn.apache.org/ant/binaries/apache-ant-1.9.16-bin.tar.bz2', 'apache-ant', '1.9.16'),
('https://inkscape.org/gallery/item/13330/inkscape-0.92.4_A6N0YOn.tar.bz2', 'inkscape', '0.92.4'),
('https://lab.nexedi.com/nexedi/userhosts/repository/a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a/archive.tar.gz', 'userhosts', 'a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a'),
]) ])
def test_namever(url, nameok, verok): def test_namever(url, nameok, verok):
assert nxdbom.namever(url) == (nameok, verok) assert nxdbom.namever(url) == (nameok, verok)
...@@ -89,7 +106,7 @@ ncurses 6.2 http://ftp.gnu.org/gnu/ncurses/ncurses-6 ...@@ -89,7 +106,7 @@ ncurses 6.2 http://ftp.gnu.org/gnu/ncurses/ncurses-6
""") """)
case1("""\ case1("""\
[neoppod-repository] [neoppod-repository.git]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/neoppod.git repository = https://lab.nexedi.com/nexedi/neoppod.git
""", """ """, """
...@@ -97,6 +114,24 @@ repository = https://lab.nexedi.com/nexedi/neoppod.git ...@@ -97,6 +114,24 @@ repository = https://lab.nexedi.com/nexedi/neoppod.git
neoppod HEAD https://lab.nexedi.com/nexedi/neoppod.git neoppod HEAD https://lab.nexedi.com/nexedi/neoppod.git
""") """)
case1("""\
[neoppod-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/neoppod
""", """
>>> gits:
neoppod HEAD https://lab.nexedi.com/nexedi/neoppod
""")
case1("""\
[neoppod-repository-github]
recipe = slapos.recipe.build:gitclone
repository = https://github.com/nexedi/neoppod
""", """
>>> gits:
neoppod HEAD https://github.com/nexedi/neoppod
""")
case1("""\ case1("""\
[ocropy-eng-traineddata] [ocropy-eng-traineddata]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
...@@ -504,6 +539,23 @@ scons-local 2.3.0 https://prdownloads.sourceforge.net/scon ...@@ -504,6 +539,23 @@ scons-local 2.3.0 https://prdownloads.sourceforge.net/scon
scons-local 2.3.1 https://prdownloads.sourceforge.net/scons/scons-local-2.3.1.tar.gz scons-local 2.3.1 https://prdownloads.sourceforge.net/scons/scons-local-2.3.1.tar.gz
""") """)
# edge case: tesseract from old ( 1.0.167.10 ) ERP5 software release
case1("""\
[tesseract-eng-traineddata]
md5sum = 57e0df3d84fed9fbf8c7a8e589f8f012
recipe = slapos.recipe.build:download
url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata
[tesseract-osd-traineddata]
md5sum = 7611737524efd1ce2dde67eff629bbcf
recipe = slapos.recipe.build:download
url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata
""",
"""\
tessdata 590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/eng.traineddata
tessdata 590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata
""")
def populate_software_directory_from_build(tmpdir, build): def populate_software_directory_from_build(tmpdir, build):
build = '-- /ROOT/.installed.cfg --\n' + build build = '-- /ROOT/.installed.cfg --\n' + build
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7