1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3 4# 5# Copyright 2017, Data61 6# Commonwealth Scientific and Industrial Research Organisation (CSIRO) 7# ABN 41 687 119 230. 8# 9# This software may be distributed and modified according to the terms of 10# the BSD 2-Clause license. Note that NO WARRANTY is provided. 11# See "LICENSE_BSD2.txt" for details. 12# 13# @TAG(DATA61_BSD) 14# 15 16from __future__ import absolute_import, division, print_function, \ 17 unicode_literals 18 19import os, sys, unittest 20 21ME = os.path.abspath(__file__) 22 23# Make CAmkES importable 24sys.path.append(os.path.join(os.path.dirname(ME), '../../..')) 25 26from camkes.internal.tests.utils import CAmkESTest, cpp_available 27from camkes.parser.stage0 import CPP, Reader 28from camkes.parser.stage1 import Parse1 29from camkes.parser.exception import ParseError 30 31class TestStage1(CAmkESTest): 32 def setUp(self): 33 super(TestStage1, self).setUp() 34 r = Reader() 35 self.parser = Parse1(r) 36 37 r = CPP() 38 self.cpp_parser = Parse1(r) 39 40 def test_empty_string(self): 41 source, content, read = self.parser.parse_string('') 42 43 self.assertEqual(source, '') 44 self.assertEqual(content.head, 'start') 45 self.assertEqual(content.tail, []) 46 self.assertLen(read, 0) 47 48 def test_basic_entity(self): 49 source, content, read = self.parser.parse_string('component foo {}') 50 51 self.assertEqual(source, 'component foo {}') 52 self.assertEqual(content.head, 'start') 53 self.assertLen(content.tail, 1) 54 self.assertEqual(content.tail[0].head, 'component_decl') 55 self.assertLen(content.tail[0].tail, 2) 56 self.assertEqual(content.tail[0].tail[0].head, 'id') 57 self.assertLen(content.tail[0].tail[0].tail, 1) 58 self.assertEqual(content.tail[0].tail[0].tail[0], 'foo') 59 self.assertEqual(content.tail[0].tail[1].head, 'component_defn') 60 self.assertLen(content.tail[0].tail[1].tail, 0) 61 self.assertLen(read, 0) 62 63 def test_numerics_basic(self): 64 source, content, read = self.parser.parse_string( 65 'configuration { hello.world = 1 + 4 - 2; }') 66 67 self.assertEqual(source, 68 'configuration { hello.world = 1 + 4 - 2; }') 69 self.assertEqual(content.head, 'start') 70 self.assertLen(content.tail, 1) 71 self.assertEqual(content.tail[0].head, 'configuration_decl') 72 73 self.assertLen(content.tail[0].tail, 1) 74 self.assertEqual(content.tail[0].tail[0].head, 'configuration_defn') 75 configuration_defn = content.tail[0].tail[0] 76 77 self.assertLen(configuration_defn.tail, 1) 78 self.assertEqual(configuration_defn.tail[0].head, 'setting') 79 setting = configuration_defn.tail[0] 80 81 self.assertLen(setting.tail, 3) 82 self.assertEqual(setting.tail[0].head, 'id') 83 self.assertLen(setting.tail[0].tail, 1) 84 self.assertEqual(setting.tail[0].tail[0], 'hello') 85 self.assertEqual(setting.tail[1].head, 'id') 86 self.assertLen(setting.tail[1].tail, 1) 87 self.assertEqual(setting.tail[1].tail[0], 'world') 88 p = setting.tail[2] 89 90 self.assertEqual(p.head, 'precedence11') 91 self.assertLen(p.tail, 1) 92 p = p.tail[0] 93 94 self.assertEqual(p.head, 'precedence10') 95 self.assertLen(p.tail, 1) 96 p = p.tail[0] 97 98 self.assertEqual(p.head, 'precedence9') 99 self.assertLen(p.tail, 1) 100 p = p.tail[0] 101 102 self.assertEqual(p.head, 'precedence8') 103 self.assertLen(p.tail, 1) 104 p = p.tail[0] 105 106 self.assertEqual(p.head, 'precedence7') 107 self.assertLen(p.tail, 1) 108 p = p.tail[0] 109 110 self.assertEqual(p.head, 'precedence6') 111 self.assertLen(p.tail, 1) 112 p = p.tail[0] 113 114 self.assertEqual(p.head, 'precedence5') 115 self.assertLen(p.tail, 1) 116 p = p.tail[0] 117 118 self.assertEqual(p.head, 'precedence4') 119 self.assertLen(p.tail, 1) 120 p = p.tail[0] 121 122 self.assertEqual(p.head, 'precedence3') 123 self.assertLen(p.tail, 1) 124 p = p.tail[0] 125 126 self.assertEqual(p.head, 'precedence2') 127 self.assertLen(p.tail, 5) 128 129 self.assertEqual(p.tail[0].head, 'precedence1') 130 self.assertLen(p.tail[0].tail, 1) 131 self.assertEqual(p.tail[0].tail[0].head, 'number') 132 self.assertLen(p.tail[0].tail[0].tail, 1) 133 self.assertEqual(p.tail[0].tail[0].tail[0], '1') 134 135 self.assertEqual(p.tail[1].head, 'add') 136 137 self.assertEqual(p.tail[2].head, 'precedence1') 138 self.assertLen(p.tail[2].tail, 1) 139 self.assertEqual(p.tail[2].tail[0].head, 'number') 140 self.assertLen(p.tail[0].tail[0].tail, 1) 141 self.assertEqual(p.tail[2].tail[0].tail[0], '4') 142 143 self.assertEqual(p.tail[3].head, 'sub') 144 145 self.assertEqual(p.tail[4].head, 'precedence1') 146 self.assertLen(p.tail[4].tail, 1) 147 self.assertEqual(p.tail[4].tail[0].head, 'number') 148 self.assertLen(p.tail[4].tail[0].tail, 1) 149 self.assertEqual(p.tail[4].tail[0].tail[0], '2') 150 151 def test_numerics_precedence(self): 152 source, content, read = self.parser.parse_string( 153 'configuration { hello.world = 1 + 4 * 2; }') 154 155 self.assertEqual(source, 156 'configuration { hello.world = 1 + 4 * 2; }') 157 158 self.assertEqual(content.head, 'start') 159 self.assertLen(content.tail, 1) 160 self.assertEqual(content.tail[0].head, 'configuration_decl') 161 162 self.assertLen(content.tail[0].tail, 1) 163 self.assertEqual(content.tail[0].tail[0].head, 'configuration_defn') 164 configuration_defn = content.tail[0].tail[0] 165 166 self.assertLen(configuration_defn.tail, 1) 167 self.assertEqual(configuration_defn.tail[0].head, 'setting') 168 setting = configuration_defn.tail[0] 169 170 self.assertLen(setting.tail, 3) 171 self.assertEqual(setting.tail[0].head, 'id') 172 self.assertLen(setting.tail[0].tail, 1) 173 self.assertEqual(setting.tail[0].tail[0], 'hello') 174 self.assertEqual(setting.tail[1].head, 'id') 175 self.assertLen(setting.tail[1].tail, 1) 176 self.assertEqual(setting.tail[1].tail[0], 'world') 177 p = setting.tail[2] 178 179 self.assertEqual(p.head, 'precedence11') 180 self.assertLen(p.tail, 1) 181 p = p.tail[0] 182 183 self.assertEqual(p.head, 'precedence10') 184 self.assertLen(p.tail, 1) 185 p = p.tail[0] 186 187 self.assertEqual(p.head, 'precedence9') 188 self.assertLen(p.tail, 1) 189 p = p.tail[0] 190 191 self.assertEqual(p.head, 'precedence8') 192 self.assertLen(p.tail, 1) 193 p = p.tail[0] 194 195 self.assertEqual(p.head, 'precedence7') 196 self.assertLen(p.tail, 1) 197 p = p.tail[0] 198 199 self.assertEqual(p.head, 'precedence6') 200 self.assertLen(p.tail, 1) 201 p = p.tail[0] 202 203 self.assertEqual(p.head, 'precedence5') 204 self.assertLen(p.tail, 1) 205 p = p.tail[0] 206 207 self.assertEqual(p.head, 'precedence4') 208 self.assertLen(p.tail, 1) 209 p = p.tail[0] 210 211 self.assertEqual(p.head, 'precedence3') 212 self.assertLen(p.tail, 1) 213 p = p.tail[0] 214 215 self.assertEqual(p.head, 'precedence2') 216 self.assertLen(p.tail, 3) 217 218 self.assertEqual(p.tail[0].head, 'precedence1') 219 self.assertLen(p.tail[0].tail, 1) 220 self.assertEqual(p.tail[0].tail[0].head, 'number') 221 self.assertLen(p.tail[0].tail[0].tail, 1) 222 self.assertEqual(p.tail[0].tail[0].tail[0], '1') 223 224 self.assertEqual(p.tail[1].head, 'add') 225 226 self.assertEqual(p.tail[2].head, 'precedence1') 227 self.assertLen(p.tail[2].tail, 3) 228 self.assertEqual(p.tail[2].tail[0].head, 'number') 229 self.assertLen(p.tail[0].tail[0].tail, 1) 230 self.assertEqual(p.tail[2].tail[0].tail[0], '4') 231 232 self.assertEqual(p.tail[2].tail[1].head, 'mul') 233 234 self.assertEqual(p.tail[2].tail[2].head, 'number') 235 self.assertLen(p.tail[2].tail[2].tail, 1) 236 self.assertEqual(p.tail[2].tail[2].tail[0], '2') 237 238 def test_malformed(self): 239 with self.assertRaises(Exception): 240 self.parser.parse_string('hello world') 241 242 def test_unicode(self): 243 source, content, read = self.parser.parse_string('component fo�� {}') 244 245 self.assertEqual(source, 'component fo�� {}') 246 self.assertEqual(content.head, 'start') 247 self.assertLen(content.tail, 1) 248 self.assertEqual(content.tail[0].head, 'component_decl') 249 self.assertLen(content.tail[0].tail, 2) 250 self.assertEqual(content.tail[0].tail[0].head, 'id') 251 self.assertLen(content.tail[0].tail[0].tail, 1) 252 self.assertEqual(content.tail[0].tail[0].tail[0], 'fo��') 253 self.assertEqual(content.tail[0].tail[1].head, 'component_defn') 254 self.assertLen(content.tail[0].tail[1].tail, 0) 255 self.assertLen(read, 0) 256 257 def test_from_file(self): 258 tmp = self.mkstemp() 259 with open(tmp, 'wt') as f: 260 f.write('component foo {}') 261 262 source, content, read = self.parser.parse_file(tmp) 263 264 self.assertEqual(source, 'component foo {}') 265 self.assertEqual(content.head, 'start') 266 self.assertLen(content.tail, 1) 267 self.assertEqual(content.tail[0].head, 'component_decl') 268 self.assertLen(content.tail[0].tail, 2) 269 self.assertEqual(content.tail[0].tail[0].head, 'id') 270 self.assertLen(content.tail[0].tail[0].tail, 1) 271 self.assertEqual(content.tail[0].tail[0].tail[0], 'foo') 272 self.assertEqual(content.tail[0].tail[1].head, 'component_defn') 273 self.assertLen(content.tail[0].tail[1].tail, 0) 274 self.assertEqual(read, set([tmp])) 275 276 @unittest.skipIf(not cpp_available(), 'CPP not found') 277 def test_with_cpp(self): 278 parent = self.mkstemp() 279 child = self.mkstemp() 280 281 with open(parent, 'wt') as f: 282 f.write('component foo\n#include "%s"' % child) 283 with open(child, 'wt') as f: 284 f.write('{}') 285 286 _, content, read = self.cpp_parser.parse_file(parent) 287 288 self.assertEqual(content.head, 'start') 289 self.assertLen(content.tail, 1) 290 self.assertEqual(content.tail[0].head, 'component_decl') 291 self.assertLen(content.tail[0].tail, 2) 292 self.assertEqual(content.tail[0].tail[0].head, 'id') 293 self.assertLen(content.tail[0].tail[0].tail, 1) 294 self.assertEqual(content.tail[0].tail[0].tail[0], 'foo') 295 self.assertEqual(content.tail[0].tail[1].head, 'component_defn') 296 self.assertLen(content.tail[0].tail[1].tail, 0) 297 298 self.assertIn(parent, read) 299 self.assertIn(child, read) 300 301 def test_lineno_basic(self): 302 _, content, _ = self.parser.parse_string('component foo {}') 303 304 self.assertEqual(content.head, 'start') 305 self.assertLen(content.tail, 1) 306 self.assertEqual(content.tail[0].head, 'component_decl') 307 component_decl = content.tail[0] 308 309 component_decl.calc_position() 310 self.assertEqual(component_decl.min_line, 1) 311 312 def test_lineno_basic2(self): 313 _, content, _ = self.parser.parse_string('\ncomponent foo {}') 314 315 self.assertEqual(content.head, 'start') 316 self.assertLen(content.tail, 1) 317 self.assertEqual(content.tail[0].head, 'component_decl') 318 component_decl = content.tail[0] 319 320 component_decl.calc_position() 321 self.assertEqual(component_decl.min_line, 2) 322 323 def test_lineno_basic3(self): 324 _, content, _ = self.parser.parse_string('component\nfoo\n{\n}') 325 326 self.assertEqual(content.head, 'start') 327 self.assertLen(content.tail, 1) 328 self.assertEqual(content.tail[0].head, 'component_decl') 329 component_decl = content.tail[0] 330 331 component_decl.calc_position() 332 self.assertIn(component_decl.min_line, (1, 2)) 333 334 def test_lineno_comment(self): 335 _, content, _ = self.parser.parse_string('//I\'m a comment\ncomponent foo {}') 336 337 self.assertEqual(content.head, 'start') 338 self.assertLen(content.tail, 1) 339 self.assertEqual(content.tail[0].head, 'component_decl') 340 component_decl = content.tail[0] 341 342 component_decl.calc_position() 343 self.assertEqual(component_decl.min_line, 2) 344 345 def test_lineno_comment2(self): 346 _, content, _ = self.parser.parse_string('/* I\'m a comment */\ncomponent foo {}') 347 348 self.assertEqual(content.head, 'start') 349 self.assertLen(content.tail, 1) 350 self.assertEqual(content.tail[0].head, 'component_decl') 351 component_decl = content.tail[0] 352 353 component_decl.calc_position() 354 self.assertEqual(component_decl.min_line, 2) 355 356 def test_lineno_multiline_comment(self): 357 _, content, _ = self.parser.parse_string('/*I\'m a \nmultiline comment*/\ncomponent foo {}') 358 359 self.assertEqual(content.head, 'start') 360 self.assertLen(content.tail, 1) 361 self.assertEqual(content.tail[0].head, 'component_decl') 362 component_decl = content.tail[0] 363 364 component_decl.calc_position() 365 self.assertEqual(component_decl.min_line, 3) 366 367 def test_lineno_multiline_comment2(self): 368 _, content, _ = self.parser.parse_string('/*I\'m\na\nmultiline\ncomment*/\ncomponent foo {}') 369 370 self.assertEqual(content.head, 'start') 371 self.assertLen(content.tail, 1) 372 self.assertEqual(content.tail[0].head, 'component_decl') 373 component_decl = content.tail[0] 374 375 component_decl.calc_position() 376 self.assertEqual(component_decl.min_line, 5) 377 378 def test_simple_spec_complete(self): 379 _, content, _ = self.parser.parse_string(''' 380 procedure Hello { 381 void hello(void); 382 } 383 384 component Foo { 385 provides Hello h; 386 } 387 388 component Bar { 389 control; 390 uses Hello w; 391 } 392 393 assembly { 394 composition { 395 component Foo f; 396 component Bar b; 397 connection Conn conn(from Foo.h, to Bar.w); 398 } 399 } 400 ''') 401 402 def test_multiline_comment_preserves_line_numbers(self): 403 _, content, _ = self.parser.parse_string(''' 404 /* multiline comment 405 * that may affect line numbering 406 * if we've messed up 407 */ 408 procedure P {} 409 ''') 410 411 self.assertEqual(content.head, 'start') 412 self.assertLen(content.tail, 1) 413 P = content.tail[0] 414 self.assertEqual(P.head, 'procedure_decl') 415 416 P.calc_position() 417 self.assertEqual(P.min_line, 6) 418 419 def test_manual_line_directives(self): 420 ''' 421 GCC supports two forms of line directive: 422 423 #line <linenum> <filename> 424 425 # <linenum> <filename> 426 427 Only the first of these is documented in [0], though it is actually the 428 second of these that the C pre processor outputs. CAmkES' understanding 429 of these directives was previously based on observed directives from 430 CPP and it could not understand the first of these. This test checks 431 that it can now understand the first of these as well, as these may be 432 in use by some other non-CPP pre processing tool. 433 434 [0]: https://gcc.gnu.org/onlinedocs/cpp/Line-Control.html 435 ''' 436 437 # Deliberately parse a malformed spec to generate an error. 438 try: 439 self.parser.parse_string(''' 440 441 #line 42 "A" 442 443 connector Foo { 444 from Procedure bar; 445 to Procedure baz; 446 } 447 ''') 448 449 # If we reached this point, the malformed spec did not trigger an 450 # error as expected. 451 self.fail('incorrect syntax accepted by stage 1 parser') 452 453 except ParseError as e: 454 error = e 455 456 self.assertRegexpMatches(str(error), 'A:44:', 'alternate form of line ' 457 'directive not supported') 458 459 def test_multiple_error_message_line_numbers(self): 460 ''' 461 This test checks that when multiple PlyPlus errors are triggered on a 462 spec that has C pre-processor line directives, the correct source line 463 is still located in the error message. There was previously an issue 464 where line number information would not be provided in this case. 465 ''' 466 467 # Parse a malformed spec with some line directives. Note that the spec 468 # contains the old form of connector definition. 469 try: 470 self.parser.parse_string(''' 471 472 # 42 "A" 473 474 connector Foo { 475 from Procedure bar; 476 to Procedure baz; 477 } 478 479 connector Qux { 480 from Procedure Moo; 481 from Procedure Cow; 482 } 483 ''') 484 485 # If we reached this point, the malformed spec did not trigger an 486 # error as expected. 487 self.fail('incorrect syntax accepted by stage 1 parser') 488 489 except ParseError as e: 490 self.assertGreaterEqual(len(e.args), 2, 'only a ' 491 'single error triggered when multiple were expected') 492 493 # If the line number narrowing algorithm has correctly taken the 494 # line directive into account, we should get the following prefix. 495 self.assertRegexpMatches(str(e), 'A:44:', 'line directive not ' 496 'accounted for') 497 498 def test_list_dict_key(self): 499 ''' 500 Test using a list as a key in a dictionary used as a setting. Should 501 not be accepted. 502 ''' 503 with self.assertRaises(ParseError): 504 self.parser.parse_string( 505 'configuration {\n' 506 ' foo.bar = {[1, 2, 3]: 3};\n' 507 '}') 508 509 def test_c_include(self): 510 ''' 511 Test we can parse a C relative path include. 512 ''' 513 _, content, _ = self.parser.parse_string( 514 'procedure Foo { include "hello.h"; }') 515 516 self.assertEqual(content.head, 'start') 517 self.assertLen(content.tail, 1) 518 519 self.assertEqual(content.tail[0].head, 'procedure_decl') 520 proc_decl = content.tail[0] 521 522 self.assertLen(proc_decl.tail, 2) 523 self.assertEqual(proc_decl.tail[0].head, 'id') 524 self.assertLen(proc_decl.tail[0].tail, 1) 525 self.assertEqual(proc_decl.tail[0].tail[0], 'Foo') 526 527 self.assertEqual(proc_decl.tail[1].head, 'procedure_defn') 528 proc_defn = proc_decl.tail[1] 529 530 self.assertLen(proc_defn.tail, 1) 531 self.assertEqual(proc_defn.tail[0].head, 'include') 532 include = proc_defn.tail[0] 533 534 self.assertLen(include.tail, 1) 535 self.assertEqual(include.tail[0].head, 'multi_string') 536 multi_string = include.tail[0] 537 538 self.assertLen(multi_string.tail, 1) 539 self.assertEqual(multi_string.tail[0].head, 'quoted_string') 540 quoted_string = multi_string.tail[0] 541 542 self.assertLen(quoted_string.tail, 1) 543 self.assertEqual(quoted_string.tail[0], '"hello.h"') 544 545if __name__ == '__main__': 546 unittest.main() 547