Mercurial > hg > freeDiameter

comparison contrib/tools/csv_to_fd @ 1461:a86eb3375b95

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add csv_to_fd, org_to_csv tools csv_to_fd converts CSV files containing RADIUS or Diameter AVP tables into various formats, including freeDiameter C code and JSON documents. org_to_csv converts org files into CSV files, suitable for csv_to_fd.
author Luke Mewburn <luke@mewburn.net>
date Mon, 09 Mar 2020 21:24:24 +1100
parents
children 8f6c77f24b1a
comparison
equal deleted inserted replaced
1460:4f44d206e60d 1461:a86eb3375b95
1 #!/usr/bin/env python
2 # vim: set fileencoding=utf-8 :
3
4 """
5 Convert CSV files containing RADIUS or Diameter AVP tables
6 into various formats.
7
8 Format of the CSV files is one of:
9 - Row per 3GPP AVP tables:
10 Name, Code, Section, DataType, Must, May, ShouldNot, MustNot [, ...]
11 - Name:
12 AVP Name. String, validated as ALPHA *(ALPHA / DIGIT / "-")
13 - Code:
14 AVP Code. Integer, 0..4294967295.
15 - Section:
16 Section in relevant standard. String.
17 - DataType:
18 AVP Data Type. String, validated per Per RFC 6733 § 4.2 and § 4.3.
19 - Must, May, ShouldNot, MustNot:
20 Flags, possibly comma or space separated: M, V
21
22 - Comment row. First cell:
23 # Comment text Comment text
24 #= Header row of ====
25 # Blank line
26
27 - Parameter row:
28 @Parameter,Value [, ...]
29 Supported Parameter terms:
30 standard Standard name. E.g. '3GPP TS 29.272', 'RFC 6733'.
31 vendor Vendor number.
32
33 """
34
35 from __future__ import print_function
36 from __future__ import with_statement
37
38 import abc
39 import csv
40 import collections
41 import json
42 import re
43 import optparse
44 import sys
45
46 CSV_COLUMN_NAMES = [
47 'name',
48 'code',
49 'section',
50 'datatype',
51 'must',
52 'may',
53 'shouldnot',
54 'mustnot',
55 'encrypt',
56 ]
57
58 VENDOR_TO_NAME = {
59 0: '',
60 193: 'Ericsson',
61 8164: 'Starent',
62 10415: '3GPP',
63 }
64
65
66 class Avp(object):
67 """Store an AVP row."""
68
69 # Regex to validate avp-name per RFC 6733 § 3.2,
70 # with changes:
71 # - Allow avp-name to start with numbers (for 3GPP)
72 # - Allow '.' in avp-name, for existing dict_dcca_3gpp usage.
73 # TODO: if starts with digit, ensure contains a letter somewhere?
74 _name_re = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*$')
75
76 __slots__ = CSV_COLUMN_NAMES + [
77 'filename', 'linenum', 'standard', 'vendor', ]
78
79 def __init__(self, name, code, section, datatype,
80 must, may, shouldnot, mustnot, encrypt,
81 filename=None, linenum=0, standard=None, vendor=0):
82 # Members from CSV row
83 self.name = name
84 self.code = int(code)
85 self.section = section
86 self.datatype = datatype
87 self.must = must
88 self.may = may
89 self.shouldnot = shouldnot
90 self.mustnot = mustnot
91 self.encrypt = encrypt
92 # Members from file state
93 self.filename = filename
94 self.linenum = linenum
95 self.standard = standard
96 self.vendor = vendor
97 # Validate CSV fields
98 if not self._name_re.match(self.name):
99 raise ValueError('Invalid AVP name "{}"'.format(self.name))
100 if (self.code < 0 or self.code > 4294967295):
101 raise ValueError('Invalid AVP code {}'.format(self.code))
102 if self.datatype not in (
103 'OctetString', 'Integer32', 'Integer64', 'Unsigned32',
104 'Unsigned64', 'Float32', 'Float64', 'Grouped',
105 'Address', 'Time', 'UTF8String', 'DiameterIdentity',
106 'DiameterURI', 'Enumerated', 'IPFilterRule',
107 ):
108 raise ValueError('Invalid AVP data type "{}"'.format(
109 self.datatype))
110 # TODO: validate must, may, shouldnot, mustnot
111
112 @property
113 def __dict__(self):
114 return {s: getattr(self, s) for s in self.__slots__}
115
116
117 class Processor(object):
118 """Interface for processor of Avp"""
119
120 __metaclass__ = abc.ABCMeta
121
122 @classmethod
123 def cls_name(cls):
124 """Return the name, lower-case, without "processor" suffix."""
125 suffix = 'processor'
126 name = cls.__name__.lower()
127 if name.endswith(suffix):
128 return name[:-len(suffix)]
129 return name
130
131 @classmethod
132 def cls_desc(cls):
133 """Return the first line of the docstring."""
134 if cls.__doc__ is None:
135 return ""
136 return cls.__doc__.split('\n')[0]
137
138 @abc.abstractmethod
139 def next_file(self, filename):
140 """Called when a file is opened."""
141 pass
142
143 @abc.abstractmethod
144 def avp(self, avp):
145 """Process a validated Avp."""
146 pass
147
148 @abc.abstractmethod
149 def comment(self, comment, filename, linenum):
150 """Process a comment row:
151 #comment,
152 """
153 pass
154
155 @abc.abstractmethod
156 def generate(self):
157 """Invoked after all rows processed."""
158 pass
159
160
161 class DebugProcessor(Processor):
162 """Display the CSV parsing"""
163
164 def next_file(self, filename):
165 print('File: {}'.format(filename))
166
167 def avp(self, avp):
168 avpdict = vars(avp)
169 print('AVP: {name}, {code}, {datatype}'.format(**avpdict))
170
171 def comment(self, comment, filename, linenum):
172 print('Comment: {}'.format(comment))
173
174 def generate(self):
175 print('Generate')
176
177
178 class NoopProcessor(Processor):
179 """Validate the CSV; no other output"""
180
181 def next_file(self, filename):
182 pass
183
184 def avp(self, avp):
185 pass
186
187 def comment(self, comment, filename, linenum):
188 pass
189
190 def generate(self):
191 pass
192
193
194 class FdcProcessor(Processor):
195 """Generate freeDiameter C code
196
197 Comment cells are parsed as:
198 # text comment /* text comment */
199 #= /*==============*/
200 # [blank line]
201 """
202
203 COMMENT_WIDTH = 64
204
205 DERIVED_TO_BASE = {
206 'Address': 'OctetString',
207 'Time': 'OctetString',
208 'UTF8String': 'OctetString',
209 'DiameterIdentity': 'OctetString',
210 'DiameterURI': 'OctetString',
211 'Enumerated': 'Integer32',
212 'IPFilterRule': 'OctetString',
213 }
214
215 def __init__(self):
216 self.lines = []
217
218 def next_file(self, filename):
219 print('/* CSV file: {} */'.format(filename))
220
221 def avp(self, avp):
222 comment = '{name}, {datatype}, code {code}'.format(**vars(avp))
223 if '' != avp.section:
224 comment += ', section {}'.format(avp.section)
225 self.add_comment(comment)
226 self.add('\t{')
227 self.add('\t\tstruct dict_avp_data data = {')
228 # TODO: remove comments?
229 self.add('\t\t\t{},\t/* Code */'.format(avp.code))
230 self.add('\t\t\t{},\t/* Vendor */'.format(avp.vendor))
231 self.add('\t\t\t\"{}\",\t/* Name */'.format(avp.name))
232 self.add('\t\t\t{},\t/* Fixed flags */'.format(
233 self.build_flags(', '.join([avp.must, avp.mustnot]))))
234 self.add('\t\t\t{},\t/* Fixed flag values */'.format(
235 self.build_flags(avp.must)))
236 # TODO: add trailing comma?
237 self.add('\t\t\tAVP_TYPE_{}\t/* base type of data */'.format(
238 self.DERIVED_TO_BASE.get(
239 avp.datatype, avp.datatype).upper()))
240 self.add('\t\t};')
241 avp_type = 'NULL'
242 if 'Enumerated' == avp.datatype:
243 self.add('\t\tstruct dict_object\t*type;')
244 vendor_prefix = ''
245 if avp.vendor != 0:
246 vendor_prefix = '{}/'.format(VENDOR_TO_NAME[avp.vendor])
247 self.add(
248 '\t\tstruct dict_type_data\t tdata = {{ AVP_TYPE_INTEGER32, '
249 '"Enumerated({prefix}{name})", NULL, NULL, NULL }};'.format(
250 prefix=vendor_prefix, name=avp.name))
251 # XXX: add enumerated values
252 self.add('\t\tCHECK_dict_new(DICT_TYPE, &tdata, NULL, &type);')
253 avp_type = "type"
254 elif avp.datatype in self.DERIVED_TO_BASE:
255 avp_type = '{}_type'.format(avp.datatype)
256 self.add('\t\tCHECK_dict_new(DICT_AVP, &data, {}, NULL);'.format(
257 avp_type))
258 # TODO: remove ; on scope brace
259 self.add('\t};')
260 self.add('')
261
262 def comment(self, comment, filename, linenum):
263 if '' == comment:
264 self.add('')
265 elif '=' == comment:
266 self.add_header()
267 elif comment.startswith(' '):
268 self.add_comment(comment[1:])
269 else:
270 raise ValueError('Unsupported comment "{}"'.format(comment))
271
272 def generate(self):
273 self.print_header()
274 self.print_comment('Start of generated data.')
275 self.print_comment('')
276 self.print_comment('The following is created automatically with:')
277 self.print_comment(' csv_to_fd -p {}'.format(self.cls_name()))
278 self.print_comment('Changes will be lost during the next update.')
279 self.print_comment('Do not modify;'
280 ' modify the source .csv file instead.')
281 self.print_header()
282 print('')
283 print('\n'.join(self.lines))
284 self.print_header()
285 self.print_comment('End of generated data.')
286 self.print_header()
287
288 def build_flags(self, flags):
289 result = []
290 if 'V' in flags:
291 result.append('AVP_FLAG_VENDOR')
292 if 'M' in flags:
293 result.append('AVP_FLAG_MANDATORY')
294 return ' |'.join(result)
295
296 def add(self, line):
297 self.lines.append(line)
298
299 def add_comment(self, comment):
300 self.lines.append(self.format_comment(comment))
301
302 def add_header(self):
303 self.lines.append(self.format_header())
304
305 def format_comment(self, comment):
306 return '\t/* {:<{width}} */'.format(comment, width=self.COMMENT_WIDTH)
307
308 def format_header(self):
309 return '\t/*={:=<{width}}=*/'.format('', width=self.COMMENT_WIDTH)
310
311 def print_comment(self, comment):
312 print(self.format_comment(comment))
313
314 def print_header(self):
315 print(self.format_header())
316
317
318 class JsonProcessor(Processor):
319 """Generate freeDiameter JSON object
320 """
321
322 def __init__(self):
323 self.avps = []
324
325 def next_file(self, filename):
326 pass
327
328 def avp(self, avp):
329 flags = collections.OrderedDict([
330 ('Must', self.build_flags(avp.must)),
331 ('MustNot', self.build_flags(avp.mustnot)),
332 ])
333 row = collections.OrderedDict([
334 ('Code', avp.code),
335 ('Flags', flags),
336 ('Name', avp.name),
337 ('Type', avp.datatype),
338 ('Vendor', avp.vendor),
339 ])
340 self.avps.append(row)
341
342 def comment(self, comment, filename, linenum):
343 pass
344
345 def generate(self):
346 doc = {"AVPs": self.avps}
347 print(json.dumps(doc, indent=2))
348
349 def build_flags(self, flags):
350 result = []
351 if 'V' in flags:
352 result.append('V')
353 if 'M' in flags:
354 result.append('M')
355 return ''.join(result)
356
357
358 def main():
359
360 # Build dict of name: NameProcessor
361 processors = {
362 cls.cls_name(): cls
363 for cls in Processor.__subclasses__()
364 }
365
366 # Build Processor name to desc
367 processor_help = '\n'.join(
368 [' {:8} {}'.format(key, processors[key].cls_desc())
369 for key in sorted(processors)])
370
371 # Custom OptionParser with improved help
372 class MyParser(optparse.OptionParser):
373 """Custom OptionParser without epilog formatting."""
374 def format_help(self, formatter=None):
375 return """\
376 {}
377 Supported PROCESSOR options:
378 {}
379 """.format(
380 optparse.OptionParser.format_help(self, formatter),
381 processor_help)
382
383 # Parse options
384 parser = MyParser(
385 description="""\
386 Convert CSV files containing RADIUS or Diameter AVP tables
387 into various formats using the specified processor PROCESSOR.
388 """)
389
390 parser.add_option(
391 '-p', '--processor',
392 default='noop',
393 help='AVP processor. One of: {}. [%default]'.format(
394 ', '.join(processors.keys())))
395 (opts, args) = parser.parse_args()
396 if len(args) < 1:
397 parser.error('Incorrect number of arguments')
398
399 # Find processor
400 try:
401 avpproc = processors[opts.processor]()
402 except KeyError as e:
403 parser.error('Unknown processor "{}"'.format(opts.processor))
404
405 # dict of [vendor][code] : Avp
406 avp_codes = collections.defaultdict(dict)
407
408 # Process files
409 for filename in args:
410 avpproc.next_file(filename)
411 with open(filename, 'r') as csvfile:
412 csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES)
413 linenum = 0
414 standard = ''
415 vendor = 0
416 for row in csvdata:
417 linenum += 1
418 try:
419 if row['name'] in (None, '', 'Attribute Name'):
420 continue
421 elif row['name'].startswith('#'):
422 comment = row['name'][1:]
423 avpproc.comment(comment, filename, linenum)
424 elif row['name'].startswith('@'):
425 parameter = row['name'][1:]
426 value = row['code']
427 if False:
428 pass
429 elif 'standard' == parameter:
430 standard = value
431 elif 'vendor' == parameter:
432 vendor = int(value)
433 else:
434 raise ValueError('Unknown parameter "{}"'.format(
435 parameter))
436 else:
437 avp = Avp(filename=filename, linenum=linenum,
438 standard=standard, vendor=vendor,
439 **row)
440 # Ensure AVP vendor/code not already defined
441 if avp.code in avp_codes[avp.vendor]:
442 conflict = avp_codes[avp.vendor][avp.code]
443 raise ValueError(
444 'AVP vendor {} code {} already present'
445 ' in file "{}" line {}'.format(
446 avp.vendor, avp.code,
447 conflict.filename, conflict.linenum))
448 avp_codes[avp.vendor][avp.code] = avp
449 # Process AVP
450 avpproc.avp(avp)
451 except ValueError as e:
452 sys.stderr.write('CSV file "{}" line {}: {}: {}\n'.format(
453 filename, linenum, e.__class__.__name__, e))
454 sys.exit(1)
455
456 # Generate result
457 avpproc.generate()
458
459
460 if '__main__' == __name__:
461 main()
462
463 # vim: set et sw=4 sts=4 :
"Welcome to our mercurial repository"