Mercurial > hg > freeDiameter

comparison contrib/tools/csv_to_fd @ 1469:48fa8d70e6ad

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
csv_to_fd: validate flags and other improvements Validate the flags. Allow more columns; ignore any past column 8 "Must Not". Add basename of filenames to the regen command.
author Luke Mewburn <luke@mewburn.net>
date Wed, 18 Mar 2020 11:40:01 +1100
parents 1404de313b85
children c0aa1e66c12e
comparison
equal deleted inserted replaced
1468:e947c6dbc096 1469:48fa8d70e6ad
4 Convert CSV files containing RADIUS or Diameter AVP tables 4 Convert CSV files containing RADIUS or Diameter AVP tables
5 into various formats. 5 into various formats.
6 6
7 Format of the CSV files is one of: 7 Format of the CSV files is one of:
8 - Row per 3GPP AVP tables: 8 - Row per 3GPP AVP tables:
9 Name, Code, Section, DataType, Must, May, ShouldNot, MustNot [, ...] 9 Name, Code, Section, DataType, Must, May, ShouldNot, MustNot [, extra]
10 - Name: 10 - Name:
11 AVP Name. String, validated as ALPHA *(ALPHA / DIGIT / "-") 11 AVP Name. String, validated as ALPHA *(ALPHA / DIGIT / "-")
12 per RFC 6733 section 3.2. 12 per RFC 6733 section 3.2.
13 - Code: 13 - Code:
14 AVP Code. Integer, 0..4294967295. 14 AVP Code. Integer, 0..4294967295.
42 import csv 42 import csv
43 import collections 43 import collections
44 import json 44 import json
45 import re 45 import re
46 import optparse 46 import optparse
47 import os
47 import sys 48 import sys
48 49
49 CSV_COLUMN_NAMES = [ 50 CSV_COLUMN_NAMES = [
50 'name', 51 'name',
51 'code', 52 'code',
53 'datatype', 54 'datatype',
54 'must', 55 'must',
55 'may', 56 'may',
56 'shouldnot', 57 'shouldnot',
57 'mustnot', 58 'mustnot',
58 'encrypt',
59 ] 59 ]
60 60
61 DERIVED_TO_BASE = { 61 DERIVED_TO_BASE = {
62 'Address': 'OctetString', # RFC 6733 section 4.3.1 62 'Address': 'OctetString', # RFC 6733 section 4.3.1
63 'Time': 'OctetString', # RFC 6733 section 4.3.1 63 'Time': 'OctetString', # RFC 6733 section 4.3.1
86 # - Allow avp-name to start with numbers (for 3GPP) 86 # - Allow avp-name to start with numbers (for 3GPP)
87 # - Allow '.' in avp-name, for existing dict_dcca_3gpp usage. 87 # - Allow '.' in avp-name, for existing dict_dcca_3gpp usage.
88 # TODO: if starts with digit, ensure contains a letter somewhere? 88 # TODO: if starts with digit, ensure contains a letter somewhere?
89 _name_re = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*$') 89 _name_re = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*$')
90 90
91 # Regex to validate flags: M, P, V, comma, space
92 _flags_re = re.compile(r'^[MPV, ]*$')
93
91 __slots__ = CSV_COLUMN_NAMES + [ 94 __slots__ = CSV_COLUMN_NAMES + [
92 'filename', 'linenum', 'standard', 'vendor', ] 95 'filename', 'linenum', 'standard', 'vendor', ]
93 96
94 def __init__(self, name, code, section, datatype, 97 def __init__(self, name, code, section, datatype,
95 must, may, shouldnot, mustnot, encrypt, 98 must, may, shouldnot, mustnot, extra_cells,
96 filename='', linenum=0, standard='', vendor=0): 99 filename='', linenum=0, standard='', vendor=0):
97 # Members from CSV row 100 # Members from CSV row
98 self.name = name 101 self.name = name
99 self.code = int(code) 102 self.code = int(code)
100 self.section = section 103 self.section = section
101 self.datatype = datatype 104 self.datatype = datatype
102 self.must = must 105 self.must = must
103 self.may = may 106 self.may = may
104 self.shouldnot = shouldnot 107 self.shouldnot = shouldnot
105 self.mustnot = mustnot 108 self.mustnot = mustnot
106 self.encrypt = encrypt
107 # Members from file state 109 # Members from file state
108 self.filename = filename 110 self.filename = filename
109 self.linenum = linenum 111 self.linenum = linenum
110 self.standard = standard 112 self.standard = standard
111 self.vendor = vendor 113 self.vendor = vendor
118 'OctetString', 'Integer32', 'Integer64', 'Unsigned32', 120 'OctetString', 'Integer32', 'Integer64', 'Unsigned32',
119 'Unsigned64', 'Float32', 'Float64', 'Grouped') 121 'Unsigned64', 'Float32', 'Float64', 'Grouped')
120 and self.datatype not in DERIVED_TO_BASE): 122 and self.datatype not in DERIVED_TO_BASE):
121 raise ValueError('Invalid AVP data type "{}"'.format( 123 raise ValueError('Invalid AVP data type "{}"'.format(
122 self.datatype)) 124 self.datatype))
123 # TODO: validate must, may, shouldnot, mustnot 125 for val, desc in [
126 (self.must, 'Must'),
127 (self.may, 'May'),
128 (self.shouldnot, 'Should Not'),
129 (self.mustnot, 'Must Not'),
130 ]:
131 if not self._flags_re.match(val):
132 raise ValueError('Invalid AVP Flags {} "{}"'.format(desc, val))
124 133
125 @property 134 @property
126 def __dict__(self): 135 def __dict__(self):
127 return {s: getattr(self, s) for s in self.__slots__} 136 return {s: getattr(self, s) for s in self.__slots__}
128 137
214 """ 223 """
215 224
216 COMMENT_WIDTH = 64 225 COMMENT_WIDTH = 64
217 226
218 def __init__(self): 227 def __init__(self):
228 self.filenames = []
219 self.lines = [] 229 self.lines = []
220 230
221 def next_file(self, filename): 231 def next_file(self, filename):
222 print('/* CSV file: {} */'.format(filename)) 232 self.filenames.append(os.path.basename(filename))
223 233
224 def avp(self, avp): 234 def avp(self, avp):
225 comment = '{name}, {datatype}, code {code}'.format(**vars(avp)) 235 comment = '{name}, {datatype}, code {code}'.format(**vars(avp))
226 if avp.section != '': 236 if avp.section != '':
227 comment += ', section {}'.format(avp.section) 237 comment += ', section {}'.format(avp.section)
274 def generate(self): 284 def generate(self):
275 self.print_header() 285 self.print_header()
276 self.print_comment('Start of generated data.') 286 self.print_comment('Start of generated data.')
277 self.print_comment('') 287 self.print_comment('')
278 self.print_comment('The following is created automatically with:') 288 self.print_comment('The following is created automatically with:')
279 self.print_comment(' csv_to_fd -p {}'.format(self.cls_name())) 289 self.print_comment(' csv_to_fd -p {} {}'.format(
290 self.cls_name(), ' '.join(self.filenames)))
280 self.print_comment('Changes will be lost during the next update.') 291 self.print_comment('Changes will be lost during the next update.')
281 self.print_comment('Do not modify;' 292 self.print_comment('Do not modify;'
282 ' modify the source .csv file instead.') 293 ' modify the source .csv file instead.')
283 self.print_header() 294 self.print_header()
284 print('') 295 print('')
409 420
410 # Process files 421 # Process files
411 for filename in args: 422 for filename in args:
412 avpproc.next_file(filename) 423 avpproc.next_file(filename)
413 with open(filename, 'r') as csvfile: 424 with open(filename, 'r') as csvfile:
414 csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES) 425 csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES,
426 restkey='extra_cells')
415 linenum = 0 427 linenum = 0
416 standard = '' 428 standard = ''
417 vendor = 0 429 vendor = 0
418 for row in csvdata: 430 for row in csvdata:
419 linenum += 1 431 linenum += 1
"Welcome to our mercurial repository"