Skip to content

Commit d96f0fe

Browse files
levkowetzlevkowetz
committed
Added a django management command to populate yang model libraries from RFCs and drafts.
 - Legacy-Id: 11110
1 parent 658a5ce commit d96f0fe

4 files changed

Lines changed: 209 additions & 5 deletions

File tree

‎ietf/settings.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@
4545

4646
ALLOWED_HOSTS = [".ietf.org", ".ietf.org.", "209.208.19.216", "4.31.198.44", ]
4747

48+
# This is used to construct the path to manage.py, in order to run management
49+
# commands, for instance in etc/cron.d/datatracker
50+
DEPLOYMENT_DIR = '/a/www/ietf-datatracker'
51+
4852
# Server name of the tools server
4953
TOOLS_SERVER = 'tools.' + IETF_DOMAIN
5054
TOOLS_SERVER_URL = 'https://' + TOOLS_SERVER
@@ -472,7 +476,7 @@ def skip_unreadable_post(record):
472476
IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
473477
IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
474478
IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
475-
IDSUBMIT_PYANG_COMMAND = 'pyang -p %(workdir)s --verbose --ietf %(model)s'
479+
IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf %(model)s'
476480

477481
IDSUBMIT_CHECKER_CLASSES = (
478482
"ietf.submit.checkers.DraftIdnitsChecker",
@@ -504,6 +508,10 @@ def skip_unreadable_post(record):
504508
IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
505509
IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
506510

511+
YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
512+
YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
513+
YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
514+
507515
XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
508516

509517
MEETING_MATERIALS_SUBMISSION_START_DAYS = -90

‎ietf/submit/checkers.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,15 @@ def check_file_txt(self, path):
138138

139139
for model in model_list:
140140
path = os.path.join(workdir, model)
141+
modpath = ':'.join([
142+
workdir,
143+
settings.YANG_RFC_MODEL_DIR,
144+
settings.YANG_DRAFT_MODEL_DIR,
145+
settings.YANG_INVAL_MODEL_DIR,
146+
])
141147
with open(path) as file:
142148
text = file.readlines()
143-
cmd = settings.IDSUBMIT_PYANG_COMMAND % {"workdir": workdir, "model": path, }
149+
cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, }
144150
code, out, err = pipe(cmd)
145151
errors = 0
146152
warnings = 0

‎ietf/submit/tests.py‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
import datetime
22
import os
33
import shutil
4+
import sys
5+
from StringIO import StringIO
6+
from pyquery import PyQuery
47

58
from django.conf import settings
6-
79
from django.core.urlresolvers import reverse as urlreverse
8-
from StringIO import StringIO
9-
from pyquery import PyQuery
10+
from django.core.management import call_command
1011

1112
import debug # pyflakes:ignore
1213

@@ -39,14 +40,21 @@ def setUp(self):
3940
os.mkdir(self.archive_dir)
4041
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir
4142

43+
self.saved_yang_model_dir = settings.YANG_DRAFT_MODEL_DIR
44+
self.yang_model_dir = os.path.abspath("tmp-yang-model-dir")
45+
os.mkdir(self.yang_model_dir)
46+
settings.YANG_DRAFT_MODEL_DIR = self.yang_model_dir
47+
4248
def tearDown(self):
4349
shutil.rmtree(self.staging_dir)
4450
shutil.rmtree(self.repository_dir)
4551
shutil.rmtree(self.archive_dir)
52+
shutil.rmtree(self.yang_model_dir)
4653
settings.IDSUBMIT_STAGING_PATH = self.saved_idsubmit_staging_path
4754
settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
4855
settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
4956
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
57+
settings.YANG_DRAFT_MODEL_DIR = self.saved_yang_model_dir
5058

5159

5260
def submission_file(self, name, rev, group, format, templatename):
@@ -235,6 +243,15 @@ def submit_new_wg(self, formats):
235243
def test_submit_new_wg_txt(self):
236244
self.submit_new_wg(["txt"])
237245

246+
# Test yang module extraction management command
247+
saved_sys_stdout = sys.stdout
248+
sys.stdout = StringIO()
249+
call_command('populate_yang_model_dirs')
250+
self.assertIn('Extracting to', sys.stdout.getvalue())
251+
sys.stdout.close()
252+
sys.stdout = saved_sys_stdout
253+
self.assertTrue(os.path.exists(os.path.join(settings.YANG_DRAFT_MODEL_DIR, "ietf-mpls@2015-10-16.yang")))
254+
238255
def text_submit_new_wg_xml(self):
239256
self.submit_new_wg(["xml"])
240257

‎ietf/utils/management/commands/populate_yang_model_dirs.py‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
import os
2+
import sys
3+
import time
4+
5+
from optparse import make_option
6+
from pathlib import Path
7+
from StringIO import StringIO
8+
from textwrap import dedent
9+
from xym import xym
10+
11+
from django.conf import settings
12+
from django.core.management.base import BaseCommand
13+
14+
class Command(BaseCommand):
15+
"""
16+
Populate the yang models repositories from drafts and RFCs.
17+
18+
Extracts yang models from RFCs (found in settings.RFC_PATH and places
19+
them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in
20+
settings.YANG_DRAFT_MODEL_DIR if valid and settings.YANG_INVAL_MODEL_DIR
21+
if not.
22+
23+
"""
24+
25+
help = dedent(__doc__).strip()
26+
27+
option_list = BaseCommand.option_list + (
28+
make_option('--clean',
29+
action='store_true', dest='clean', default=False,
30+
help='Remove the current directory content before writing new models.'),
31+
)
32+
33+
def handle(self, *filenames, **options):
34+
"""
35+
36+
* All yang modules from published RFCs should be extracted and be
37+
available in an rfc-yang repository.
38+
39+
* All valid yang modules from active, not replaced, internet drafts
40+
should be extracted and be available in a draft-valid-yang repository.
41+
42+
* All, valid and invalid, yang modules from active, not replaced,
43+
internet drafts should be available in a draft-all-yang repository.
44+
(Actually, given precedence ordering, it would be enough to place
45+
non-validating modules in a draft-invalid-yang repository instead).
46+
47+
* In all cases, example modules should be excluded.
48+
49+
* Precedence is established by the search order of the repository as
50+
provided to pyang.
51+
52+
* As drafts expire, models should be removed in order to catch cases
53+
where a module being worked on depends on one which has slipped out
54+
of the work queue.
55+
56+
"""
57+
58+
verbosity = int(options.get('verbosity'))
59+
60+
def extract_from(file, dir, strict=True):
61+
saved_stderr = sys.stderr
62+
sys.stderr = StringIO()
63+
model_list = []
64+
try:
65+
model_list = xym.xym(str(item), str(item.parent), str(dir), strict=strict, debug_level=(verbosity>1))
66+
for name in model_list:
67+
modfile = moddir / name
68+
mtime = item.stat().st_mtime
69+
os.utime(str(modfile), (mtime, mtime))
70+
if '"' in name:
71+
name = name.replace('"', '')
72+
modfile.rename(str(moddir/name))
73+
model_list = [ n.replace('"','') for n in model_list ]
74+
except Exception as e:
75+
print("** Error when extracting from %s: %s" % (item, str(e)))
76+
sys.stderr = saved_stderr
77+
return model_list
78+
79+
# Extract from new RFCs
80+
81+
rfcdir = Path(settings.RFC_PATH)
82+
83+
moddir = Path(settings.YANG_RFC_MODEL_DIR)
84+
if not moddir.exists():
85+
moddir.mkdir(parents=True)
86+
87+
latest = 0
88+
for item in moddir.iterdir():
89+
if item.stat().st_mtime > latest:
90+
latest = item.stat().st_mtime
91+
92+
print("Extracting to %s ..." % moddir)
93+
for item in rfcdir.iterdir():
94+
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt'):
95+
if item.stat().st_mtime > latest:
96+
model_list = extract_from(item, moddir)
97+
for name in model_list:
98+
if name.startswith('ietf') or name.startswith('iana'):
99+
if verbosity > 1:
100+
print(" Extracted from %s: %s" % (item, name))
101+
else:
102+
sys.stdout.write('.')
103+
sys.stdout.flush()
104+
else:
105+
modfile = moddir / name
106+
modfile.unlink()
107+
if verbosity > 1:
108+
print(" Skipped module from %s: %s" % (item, name))
109+
print("")
110+
111+
# Extract valid modules from drafts
112+
113+
six_months_ago = time.time() - 6*31*24*60*60
114+
def active(item):
115+
return item.stat().st_mtime > six_months_ago
116+
117+
draftdir = Path(settings.INTERNET_DRAFT_PATH)
118+
119+
moddir = Path(settings.YANG_DRAFT_MODEL_DIR)
120+
if not moddir.exists():
121+
moddir.mkdir(parents=True)
122+
print("Emptying %s ..." % moddir)
123+
for item in moddir.iterdir():
124+
item.unlink()
125+
126+
print("Extracting to %s ..." % moddir)
127+
for item in draftdir.iterdir():
128+
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
129+
model_list = extract_from(item, moddir)
130+
for name in model_list:
131+
if not name.startswith('example'):
132+
if verbosity > 1:
133+
print(" Extracted valid module from %s: %s" % (item, name))
134+
else:
135+
sys.stdout.write('.')
136+
sys.stdout.flush()
137+
else:
138+
modfile = moddir / name
139+
modfile.unlink()
140+
if verbosity > 1:
141+
print(" Skipped module from %s: %s" % (item, name))
142+
print("")
143+
144+
# Extract invalid modules from drafts
145+
valdir = moddir
146+
moddir = Path(settings.YANG_INVAL_MODEL_DIR)
147+
if not moddir.exists():
148+
moddir.mkdir(parents=True)
149+
print("Emptying %s ..." % moddir)
150+
for item in moddir.iterdir():
151+
item.unlink()
152+
153+
print("Extracting to %s ..." % moddir)
154+
for item in draftdir.iterdir():
155+
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
156+
model_list = extract_from(item, moddir, strict=False)
157+
for name in model_list:
158+
modfile = moddir / name
159+
if (valdir/name).exists():
160+
modfile.unlink()
161+
if verbosity > 1:
162+
print(" Skipped valid module from %s: %s" % (item, name))
163+
elif not name.startswith('example'):
164+
if verbosity > 1:
165+
print(" Extracted invalid module from %s: %s" % (item, name))
166+
else:
167+
sys.stdout.write('.')
168+
sys.stdout.flush()
169+
else:
170+
modfile.unlink()
171+
if verbosity > 1:
172+
print(" Skipped module from %s: %s" % (item, name))
173+
print("")

0 commit comments

Comments
 (0)