Mercurial > hg > freeDiameter

comparison contrib/tools/csv_to_fd @ 1479:c0aa1e66c12e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
csv_to_fd: improve validation * Validate flags: - Ensure M and V can't occur more than once. - Ensure absence of V for vendor 0. - Ensure presence of V for vendor not 0. * Ensure AVP name is unique per vendor. * Add AVP name to most errors. * Use csv.DictReader.line_num for more accurate line numbers. TODO: enforce M and V must be present once and only once. Some AVPs currently fail that rule, including: - RFC 4005 / RFC 7155 QoS-Filter-Rule - 3GPP TS 29.212 PDN-Connection-ID - 3GPP TS 29.212 PS-to-CS-Session-Continuity
author Luke Mewburn <luke@mewburn.net>
date Thu, 26 Mar 2020 10:28:16 +1100
parents 48fa8d70e6ad
children 5c2d061a8c8e
comparison
equal deleted inserted replaced
1478:47ce98681bd7 1479:c0aa1e66c12e
21 - RFC 7155 section 4.1 21 - RFC 7155 section 4.1
22 - Must, May, ShouldNot, MustNot: 22 - Must, May, ShouldNot, MustNot:
23 Flags, possibly comma or space separated: M, V 23 Flags, possibly comma or space separated: M, V
24 24
25 - Comment row. First cell: 25 - Comment row. First cell:
26 # Comment text Comment text 26 # Comment text 'Comment text'
27 #= Header row of ==== 27 #= '/*========*/'
28 # Blank line 28 # Blank line
29 29
30 - Parameter row: 30 - Parameter row:
31 @Parameter,Value [, ...] 31 @Parameter,Value [, ...]
32 Supported Parameter terms: 32 Supported Parameter terms:
90 90
91 # Regex to validate flags: M, P, V, comma, space 91 # Regex to validate flags: M, P, V, comma, space
92 _flags_re = re.compile(r'^[MPV, ]*$') 92 _flags_re = re.compile(r'^[MPV, ]*$')
93 93
94 __slots__ = CSV_COLUMN_NAMES + [ 94 __slots__ = CSV_COLUMN_NAMES + [
95 'filename', 'linenum', 'standard', 'vendor', ] 95 'filename', 'line_num', 'standard', 'vendor', ]
96 96
97 def __init__(self, name, code, section, datatype, 97 def __init__(self, name, code, section, datatype,
98 must, may, shouldnot, mustnot, extra_cells, 98 must, may, shouldnot, mustnot, extra_cells=[],
99 filename='', linenum=0, standard='', vendor=0): 99 filename='', line_num=0, standard='', vendor=0):
100 # Members from CSV row 100 # Members from CSV row
101 self.name = name 101 self.name = name
102 self.code = int(code) 102 self.code = int(code)
103 self.section = section 103 self.section = section
104 self.datatype = datatype 104 self.datatype = datatype
106 self.may = may 106 self.may = may
107 self.shouldnot = shouldnot 107 self.shouldnot = shouldnot
108 self.mustnot = mustnot 108 self.mustnot = mustnot
109 # Members from file state 109 # Members from file state
110 self.filename = filename 110 self.filename = filename
111 self.linenum = linenum 111 self.line_num = line_num
112 self.standard = standard 112 self.standard = standard
113 self.vendor = vendor 113 self.vendor = vendor
114 # Validate CSV fields 114 # Validate CSV fields
115 if not self._name_re.match(self.name): 115 if not self._name_re.match(self.name):
116 raise ValueError('Invalid AVP name "{}"'.format(self.name)) 116 raise ValueError('Invalid AVP name "{}"'.format(self.name))
117 if (self.code < 0 or self.code > 4294967295): 117 if (self.code < 0 or self.code > 4294967295):
118 raise ValueError('Invalid AVP code {}'.format(self.code)) 118 raise ValueError('AVP "{}" invalid code {}'.format(
119 self.name, self.code))
119 if (self.datatype not in ( 120 if (self.datatype not in (
120 'OctetString', 'Integer32', 'Integer64', 'Unsigned32', 121 'OctetString', 'Integer32', 'Integer64', 'Unsigned32',
121 'Unsigned64', 'Float32', 'Float64', 'Grouped') 122 'Unsigned64', 'Float32', 'Float64', 'Grouped')
122 and self.datatype not in DERIVED_TO_BASE): 123 and self.datatype not in DERIVED_TO_BASE):
123 raise ValueError('Invalid AVP data type "{}"'.format( 124 raise ValueError('AVP "{}" invalid data type "{}"'.format(
124 self.datatype)) 125 self.name, self.datatype))
126 # Validate flags
127 flags = collections.Counter()
125 for val, desc in [ 128 for val, desc in [
126 (self.must, 'Must'), 129 (self.must, 'Must'),
127 (self.may, 'May'), 130 (self.may, 'May'),
128 (self.shouldnot, 'Should Not'), 131 (self.shouldnot, 'Should Not'),
129 (self.mustnot, 'Must Not'), 132 (self.mustnot, 'Must Not'),
130 ]: 133 ]:
131 if not self._flags_re.match(val): 134 if not self._flags_re.match(val):
132 raise ValueError('Invalid AVP Flags {} "{}"'.format(desc, val)) 135 raise ValueError('AVP "{}" invalid {} Flags "{}"'.format(
136 self.name, desc, val))
137 flags.update(val)
138 # Check occurrence of M,V in Must,May,ShouldNot,MustNot
139 for flag in 'MV':
140 # TODO: can AVP flags not appear at all?
141 # if flags[flag] == 0:
142 # raise ValueError('AVP "{}" Flag "{}" not set'.format(
143 # self.name, flag))
144 if flags[flag] > 1:
145 raise ValueError('AVP "{}" Flag "{}" set {} times'.format(
146 self.name, flag, flags[flag]))
147 # Compare V presence against vendor
148 if 'V' in self.must:
149 if self.vendor == 0:
150 raise ValueError('AVP "{}" Flag "V" set for vendor 0'.format(
151 self.name))
152 else:
153 if self.vendor != 0:
154 raise ValueError(
155 'AVP "{}" Flag "V" not set for vendor {}'.format(
156 self.name, self.vendor))
133 157
134 @property 158 @property
135 def __dict__(self): 159 def __dict__(self):
136 return {s: getattr(self, s) for s in self.__slots__} 160 return {s: getattr(self, s) for s in self.__slots__}
137 161
166 def avp(self, avp): 190 def avp(self, avp):
167 """Process a validated Avp.""" 191 """Process a validated Avp."""
168 pass 192 pass
169 193
170 @abc.abstractmethod 194 @abc.abstractmethod
171 def comment(self, comment, filename, linenum): 195 def comment(self, comment, filename, line_num):
172 """Process a comment row: 196 """Process a comment row:
173 #comment, 197 #comment,
174 """ 198 """
175 pass 199 pass
176 200
188 212
189 def avp(self, avp): 213 def avp(self, avp):
190 avpdict = vars(avp) 214 avpdict = vars(avp)
191 print('AVP: {name}, {code}, {datatype}'.format(**avpdict)) 215 print('AVP: {name}, {code}, {datatype}'.format(**avpdict))
192 216
193 def comment(self, comment, filename, linenum): 217 def comment(self, comment, filename, line_num):
194 print('Comment: {}'.format(comment)) 218 print('Comment: {}'.format(comment))
195 219
196 def generate(self): 220 def generate(self):
197 print('Generate') 221 print('Generate')
198 222
204 pass 228 pass
205 229
206 def avp(self, avp): 230 def avp(self, avp):
207 pass 231 pass
208 232
209 def comment(self, comment, filename, linenum): 233 def comment(self, comment, filename, line_num):
210 pass 234 pass
211 235
212 def generate(self): 236 def generate(self):
213 pass 237 pass
214 238
269 avp_type)) 293 avp_type))
270 # TODO: remove ; on scope brace 294 # TODO: remove ; on scope brace
271 self.add('\t};') 295 self.add('\t};')
272 self.add('') 296 self.add('')
273 297
274 def comment(self, comment, filename, linenum): 298 def comment(self, comment, filename, line_num):
275 if comment == '': 299 if comment == '':
276 self.add('') 300 self.add('')
277 elif comment == '=': 301 elif comment == '=':
278 self.add_header() 302 self.add_header()
279 elif comment.startswith(' '): 303 elif comment.startswith(' '):
350 ('Type', avp.datatype), 374 ('Type', avp.datatype),
351 ('Vendor', avp.vendor), 375 ('Vendor', avp.vendor),
352 ]) 376 ])
353 self.avps.append(row) 377 self.avps.append(row)
354 378
355 def comment(self, comment, filename, linenum): 379 def comment(self, comment, filename, line_num):
356 pass 380 pass
357 381
358 def generate(self): 382 def generate(self):
359 doc = {"AVPs": self.avps} 383 doc = {"AVPs": self.avps}
360 print(json.dumps(doc, indent=2)) 384 print(json.dumps(doc, indent=2))
416 parser.error('Unknown processor "{}"'.format(opts.processor)) 440 parser.error('Unknown processor "{}"'.format(opts.processor))
417 441
418 # dict of [vendor][code] : Avp 442 # dict of [vendor][code] : Avp
419 avp_codes = collections.defaultdict(dict) 443 avp_codes = collections.defaultdict(dict)
420 444
445 # dict of [vendor][name] : Avp
446 avp_names = collections.defaultdict(dict)
447
421 # Process files 448 # Process files
422 for filename in args: 449 for filename in args:
423 avpproc.next_file(filename) 450 avpproc.next_file(filename)
424 with open(filename, 'r') as csvfile: 451 with open(filename, 'r') as csvfile:
425 csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES, 452 csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES,
426 restkey='extra_cells') 453 restkey='extra_cells')
427 linenum = 0
428 standard = '' 454 standard = ''
429 vendor = 0 455 vendor = 0
430 for row in csvdata: 456 for row in csvdata:
431 linenum += 1
432 try: 457 try:
433 if row['name'] in (None, '', 'Attribute Name'): 458 if row['name'] in (None, '', 'Attribute Name'):
434 continue 459 continue
435 elif row['name'].startswith('#'): 460 elif row['name'].startswith('#'):
436 comment = row['name'][1:] 461 comment = row['name'][1:]
437 avpproc.comment(comment, filename, linenum) 462 avpproc.comment(comment, filename, csvdata.line_num)
438 elif row['name'].startswith('@'): 463 elif row['name'].startswith('@'):
439 parameter = row['name'][1:] 464 parameter = row['name'][1:]
440 value = row['code'] 465 value = row['code']
441 if False: 466 if False:
442 pass 467 pass
446 vendor = int(value) 471 vendor = int(value)
447 else: 472 else:
448 raise ValueError('Unknown parameter "{}"'.format( 473 raise ValueError('Unknown parameter "{}"'.format(
449 parameter)) 474 parameter))
450 else: 475 else:
451 avp = Avp(filename=filename, linenum=linenum, 476 avp = Avp(filename=filename, line_num=csvdata.line_num,
452 standard=standard, vendor=vendor, 477 standard=standard, vendor=vendor,
453 **row) 478 **row)
454 # Ensure AVP vendor/code not already defined 479 # Ensure AVP vendor/code not already defined
455 if avp.code in avp_codes[avp.vendor]: 480 if avp.code in avp_codes[avp.vendor]:
456 conflict = avp_codes[avp.vendor][avp.code] 481 conflict = avp_codes[avp.vendor][avp.code]
457 raise ValueError( 482 raise ValueError(
458 'AVP vendor {} code {} already present' 483 'AVP vendor {} code {} already present'
459 ' in file "{}" line {}'.format( 484 ' in file "{}" line {}'.format(
460 avp.vendor, avp.code, 485 avp.vendor, avp.code,
461 conflict.filename, conflict.linenum)) 486 conflict.filename, conflict.line_num))
462 avp_codes[avp.vendor][avp.code] = avp 487 avp_codes[avp.vendor][avp.code] = avp
488 # Ensure AVP vendor/name not already defined
489 if avp.name in avp_names[avp.vendor]:
490 conflict = avp_names[avp.vendor][avp.name]
491 raise ValueError(
492 'AVP vendor {} name "{}" already present'
493 ' in file "{}" line {}'.format(
494 avp.vendor, avp.name,
495 conflict.filename, conflict.line_num))
496 avp_names[avp.vendor][avp.name] = avp
463 # Process AVP 497 # Process AVP
464 avpproc.avp(avp) 498 avpproc.avp(avp)
465 except ValueError as e: 499 except ValueError as e:
466 sys.stderr.write('CSV file "{}" line {}: {}: {}\n'.format( 500 sys.stderr.write('CSV file "{}" line {}: {}\n'.format(
467 filename, linenum, e.__class__.__name__, e)) 501 filename, csvdata.line_num, e))
468 sys.exit(1) 502 sys.exit(1)
469 503
470 # Generate result 504 # Generate result
471 avpproc.generate() 505 avpproc.generate()
472 506
"Welcome to our mercurial repository"