1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4#
5# Copyright 2017, Data61
6# Commonwealth Scientific and Industrial Research Organisation (CSIRO)
7# ABN 41 687 119 230.
8#
9# This software may be distributed and modified according to the terms of
10# the BSD 2-Clause license. Note that NO WARRANTY is provided.
11# See "LICENSE_BSD2.txt" for details.
12#
13# @TAG(DATA61_BSD)
14#
15
16from __future__ import absolute_import, division, print_function, \
17    unicode_literals
18
19import os, sys, unittest
20
21ME = os.path.abspath(__file__)
22
23# Make CAmkES importable
24sys.path.append(os.path.join(os.path.dirname(ME), '../../..'))
25
26from camkes.internal.tests.utils import CAmkESTest, cpp_available
27from camkes.parser.stage0 import CPP, Reader
28from camkes.parser.stage1 import Parse1
29from camkes.parser.exception import ParseError
30
31class TestStage1(CAmkESTest):
32    def setUp(self):
33        super(TestStage1, self).setUp()
34        r = Reader()
35        self.parser = Parse1(r)
36
37        r = CPP()
38        self.cpp_parser = Parse1(r)
39
40    def test_empty_string(self):
41        source, content, read = self.parser.parse_string('')
42
43        self.assertEqual(source, '')
44        self.assertEqual(content.head, 'start')
45        self.assertEqual(content.tail, [])
46        self.assertLen(read, 0)
47
48    def test_basic_entity(self):
49        source, content, read = self.parser.parse_string('component foo {}')
50
51        self.assertEqual(source, 'component foo {}')
52        self.assertEqual(content.head, 'start')
53        self.assertLen(content.tail, 1)
54        self.assertEqual(content.tail[0].head, 'component_decl')
55        self.assertLen(content.tail[0].tail, 2)
56        self.assertEqual(content.tail[0].tail[0].head, 'id')
57        self.assertLen(content.tail[0].tail[0].tail, 1)
58        self.assertEqual(content.tail[0].tail[0].tail[0], 'foo')
59        self.assertEqual(content.tail[0].tail[1].head, 'component_defn')
60        self.assertLen(content.tail[0].tail[1].tail, 0)
61        self.assertLen(read, 0)
62
63    def test_numerics_basic(self):
64        source, content, read = self.parser.parse_string(
65            'configuration { hello.world = 1 + 4 - 2; }')
66
67        self.assertEqual(source,
68            'configuration { hello.world = 1 + 4 - 2; }')
69        self.assertEqual(content.head, 'start')
70        self.assertLen(content.tail, 1)
71        self.assertEqual(content.tail[0].head, 'configuration_decl')
72
73        self.assertLen(content.tail[0].tail, 1)
74        self.assertEqual(content.tail[0].tail[0].head, 'configuration_defn')
75        configuration_defn = content.tail[0].tail[0]
76
77        self.assertLen(configuration_defn.tail, 1)
78        self.assertEqual(configuration_defn.tail[0].head, 'setting')
79        setting = configuration_defn.tail[0]
80
81        self.assertLen(setting.tail, 3)
82        self.assertEqual(setting.tail[0].head, 'id')
83        self.assertLen(setting.tail[0].tail, 1)
84        self.assertEqual(setting.tail[0].tail[0], 'hello')
85        self.assertEqual(setting.tail[1].head, 'id')
86        self.assertLen(setting.tail[1].tail, 1)
87        self.assertEqual(setting.tail[1].tail[0], 'world')
88        p = setting.tail[2]
89
90        self.assertEqual(p.head, 'precedence11')
91        self.assertLen(p.tail, 1)
92        p = p.tail[0]
93
94        self.assertEqual(p.head, 'precedence10')
95        self.assertLen(p.tail, 1)
96        p = p.tail[0]
97
98        self.assertEqual(p.head, 'precedence9')
99        self.assertLen(p.tail, 1)
100        p = p.tail[0]
101
102        self.assertEqual(p.head, 'precedence8')
103        self.assertLen(p.tail, 1)
104        p = p.tail[0]
105
106        self.assertEqual(p.head, 'precedence7')
107        self.assertLen(p.tail, 1)
108        p = p.tail[0]
109
110        self.assertEqual(p.head, 'precedence6')
111        self.assertLen(p.tail, 1)
112        p = p.tail[0]
113
114        self.assertEqual(p.head, 'precedence5')
115        self.assertLen(p.tail, 1)
116        p = p.tail[0]
117
118        self.assertEqual(p.head, 'precedence4')
119        self.assertLen(p.tail, 1)
120        p = p.tail[0]
121
122        self.assertEqual(p.head, 'precedence3')
123        self.assertLen(p.tail, 1)
124        p = p.tail[0]
125
126        self.assertEqual(p.head, 'precedence2')
127        self.assertLen(p.tail, 5)
128
129        self.assertEqual(p.tail[0].head, 'precedence1')
130        self.assertLen(p.tail[0].tail, 1)
131        self.assertEqual(p.tail[0].tail[0].head, 'number')
132        self.assertLen(p.tail[0].tail[0].tail, 1)
133        self.assertEqual(p.tail[0].tail[0].tail[0], '1')
134
135        self.assertEqual(p.tail[1].head, 'add')
136
137        self.assertEqual(p.tail[2].head, 'precedence1')
138        self.assertLen(p.tail[2].tail, 1)
139        self.assertEqual(p.tail[2].tail[0].head, 'number')
140        self.assertLen(p.tail[0].tail[0].tail, 1)
141        self.assertEqual(p.tail[2].tail[0].tail[0], '4')
142
143        self.assertEqual(p.tail[3].head, 'sub')
144
145        self.assertEqual(p.tail[4].head, 'precedence1')
146        self.assertLen(p.tail[4].tail, 1)
147        self.assertEqual(p.tail[4].tail[0].head, 'number')
148        self.assertLen(p.tail[4].tail[0].tail, 1)
149        self.assertEqual(p.tail[4].tail[0].tail[0], '2')
150
151    def test_numerics_precedence(self):
152        source, content, read = self.parser.parse_string(
153            'configuration { hello.world = 1 + 4 * 2; }')
154
155        self.assertEqual(source,
156            'configuration { hello.world = 1 + 4 * 2; }')
157
158        self.assertEqual(content.head, 'start')
159        self.assertLen(content.tail, 1)
160        self.assertEqual(content.tail[0].head, 'configuration_decl')
161
162        self.assertLen(content.tail[0].tail, 1)
163        self.assertEqual(content.tail[0].tail[0].head, 'configuration_defn')
164        configuration_defn = content.tail[0].tail[0]
165
166        self.assertLen(configuration_defn.tail, 1)
167        self.assertEqual(configuration_defn.tail[0].head, 'setting')
168        setting = configuration_defn.tail[0]
169
170        self.assertLen(setting.tail, 3)
171        self.assertEqual(setting.tail[0].head, 'id')
172        self.assertLen(setting.tail[0].tail, 1)
173        self.assertEqual(setting.tail[0].tail[0], 'hello')
174        self.assertEqual(setting.tail[1].head, 'id')
175        self.assertLen(setting.tail[1].tail, 1)
176        self.assertEqual(setting.tail[1].tail[0], 'world')
177        p = setting.tail[2]
178
179        self.assertEqual(p.head, 'precedence11')
180        self.assertLen(p.tail, 1)
181        p = p.tail[0]
182
183        self.assertEqual(p.head, 'precedence10')
184        self.assertLen(p.tail, 1)
185        p = p.tail[0]
186
187        self.assertEqual(p.head, 'precedence9')
188        self.assertLen(p.tail, 1)
189        p = p.tail[0]
190
191        self.assertEqual(p.head, 'precedence8')
192        self.assertLen(p.tail, 1)
193        p = p.tail[0]
194
195        self.assertEqual(p.head, 'precedence7')
196        self.assertLen(p.tail, 1)
197        p = p.tail[0]
198
199        self.assertEqual(p.head, 'precedence6')
200        self.assertLen(p.tail, 1)
201        p = p.tail[0]
202
203        self.assertEqual(p.head, 'precedence5')
204        self.assertLen(p.tail, 1)
205        p = p.tail[0]
206
207        self.assertEqual(p.head, 'precedence4')
208        self.assertLen(p.tail, 1)
209        p = p.tail[0]
210
211        self.assertEqual(p.head, 'precedence3')
212        self.assertLen(p.tail, 1)
213        p = p.tail[0]
214
215        self.assertEqual(p.head, 'precedence2')
216        self.assertLen(p.tail, 3)
217
218        self.assertEqual(p.tail[0].head, 'precedence1')
219        self.assertLen(p.tail[0].tail, 1)
220        self.assertEqual(p.tail[0].tail[0].head, 'number')
221        self.assertLen(p.tail[0].tail[0].tail, 1)
222        self.assertEqual(p.tail[0].tail[0].tail[0], '1')
223
224        self.assertEqual(p.tail[1].head, 'add')
225
226        self.assertEqual(p.tail[2].head, 'precedence1')
227        self.assertLen(p.tail[2].tail, 3)
228        self.assertEqual(p.tail[2].tail[0].head, 'number')
229        self.assertLen(p.tail[0].tail[0].tail, 1)
230        self.assertEqual(p.tail[2].tail[0].tail[0], '4')
231
232        self.assertEqual(p.tail[2].tail[1].head, 'mul')
233
234        self.assertEqual(p.tail[2].tail[2].head, 'number')
235        self.assertLen(p.tail[2].tail[2].tail, 1)
236        self.assertEqual(p.tail[2].tail[2].tail[0], '2')
237
238    def test_malformed(self):
239        with self.assertRaises(Exception):
240            self.parser.parse_string('hello world')
241
242    def test_unicode(self):
243        source, content, read = self.parser.parse_string('component fo�� {}')
244
245        self.assertEqual(source, 'component fo�� {}')
246        self.assertEqual(content.head, 'start')
247        self.assertLen(content.tail, 1)
248        self.assertEqual(content.tail[0].head, 'component_decl')
249        self.assertLen(content.tail[0].tail, 2)
250        self.assertEqual(content.tail[0].tail[0].head, 'id')
251        self.assertLen(content.tail[0].tail[0].tail, 1)
252        self.assertEqual(content.tail[0].tail[0].tail[0], 'fo��')
253        self.assertEqual(content.tail[0].tail[1].head, 'component_defn')
254        self.assertLen(content.tail[0].tail[1].tail, 0)
255        self.assertLen(read, 0)
256
257    def test_from_file(self):
258        tmp = self.mkstemp()
259        with open(tmp, 'wt') as f:
260            f.write('component foo {}')
261
262        source, content, read = self.parser.parse_file(tmp)
263
264        self.assertEqual(source, 'component foo {}')
265        self.assertEqual(content.head, 'start')
266        self.assertLen(content.tail, 1)
267        self.assertEqual(content.tail[0].head, 'component_decl')
268        self.assertLen(content.tail[0].tail, 2)
269        self.assertEqual(content.tail[0].tail[0].head, 'id')
270        self.assertLen(content.tail[0].tail[0].tail, 1)
271        self.assertEqual(content.tail[0].tail[0].tail[0], 'foo')
272        self.assertEqual(content.tail[0].tail[1].head, 'component_defn')
273        self.assertLen(content.tail[0].tail[1].tail, 0)
274        self.assertEqual(read, set([tmp]))
275
276    @unittest.skipIf(not cpp_available(), 'CPP not found')
277    def test_with_cpp(self):
278        parent = self.mkstemp()
279        child = self.mkstemp()
280
281        with open(parent, 'wt') as f:
282            f.write('component foo\n#include "%s"' % child)
283        with open(child, 'wt') as f:
284            f.write('{}')
285
286        _, content, read = self.cpp_parser.parse_file(parent)
287
288        self.assertEqual(content.head, 'start')
289        self.assertLen(content.tail, 1)
290        self.assertEqual(content.tail[0].head, 'component_decl')
291        self.assertLen(content.tail[0].tail, 2)
292        self.assertEqual(content.tail[0].tail[0].head, 'id')
293        self.assertLen(content.tail[0].tail[0].tail, 1)
294        self.assertEqual(content.tail[0].tail[0].tail[0], 'foo')
295        self.assertEqual(content.tail[0].tail[1].head, 'component_defn')
296        self.assertLen(content.tail[0].tail[1].tail, 0)
297
298        self.assertIn(parent, read)
299        self.assertIn(child, read)
300
301    def test_lineno_basic(self):
302        _, content, _ = self.parser.parse_string('component foo {}')
303
304        self.assertEqual(content.head, 'start')
305        self.assertLen(content.tail, 1)
306        self.assertEqual(content.tail[0].head, 'component_decl')
307        component_decl = content.tail[0]
308
309        component_decl.calc_position()
310        self.assertEqual(component_decl.min_line, 1)
311
312    def test_lineno_basic2(self):
313        _, content, _ = self.parser.parse_string('\ncomponent foo {}')
314
315        self.assertEqual(content.head, 'start')
316        self.assertLen(content.tail, 1)
317        self.assertEqual(content.tail[0].head, 'component_decl')
318        component_decl = content.tail[0]
319
320        component_decl.calc_position()
321        self.assertEqual(component_decl.min_line, 2)
322
323    def test_lineno_basic3(self):
324        _, content, _ = self.parser.parse_string('component\nfoo\n{\n}')
325
326        self.assertEqual(content.head, 'start')
327        self.assertLen(content.tail, 1)
328        self.assertEqual(content.tail[0].head, 'component_decl')
329        component_decl = content.tail[0]
330
331        component_decl.calc_position()
332        self.assertIn(component_decl.min_line, (1, 2))
333
334    def test_lineno_comment(self):
335        _, content, _ = self.parser.parse_string('//I\'m a comment\ncomponent foo {}')
336
337        self.assertEqual(content.head, 'start')
338        self.assertLen(content.tail, 1)
339        self.assertEqual(content.tail[0].head, 'component_decl')
340        component_decl = content.tail[0]
341
342        component_decl.calc_position()
343        self.assertEqual(component_decl.min_line, 2)
344
345    def test_lineno_comment2(self):
346        _, content, _ = self.parser.parse_string('/* I\'m a comment */\ncomponent foo {}')
347
348        self.assertEqual(content.head, 'start')
349        self.assertLen(content.tail, 1)
350        self.assertEqual(content.tail[0].head, 'component_decl')
351        component_decl = content.tail[0]
352
353        component_decl.calc_position()
354        self.assertEqual(component_decl.min_line, 2)
355
356    def test_lineno_multiline_comment(self):
357        _, content, _ = self.parser.parse_string('/*I\'m a \nmultiline comment*/\ncomponent foo {}')
358
359        self.assertEqual(content.head, 'start')
360        self.assertLen(content.tail, 1)
361        self.assertEqual(content.tail[0].head, 'component_decl')
362        component_decl = content.tail[0]
363
364        component_decl.calc_position()
365        self.assertEqual(component_decl.min_line, 3)
366
367    def test_lineno_multiline_comment2(self):
368        _, content, _ = self.parser.parse_string('/*I\'m\na\nmultiline\ncomment*/\ncomponent foo {}')
369
370        self.assertEqual(content.head, 'start')
371        self.assertLen(content.tail, 1)
372        self.assertEqual(content.tail[0].head, 'component_decl')
373        component_decl = content.tail[0]
374
375        component_decl.calc_position()
376        self.assertEqual(component_decl.min_line, 5)
377
378    def test_simple_spec_complete(self):
379        _, content, _ = self.parser.parse_string('''
380            procedure Hello {
381                void hello(void);
382            }
383
384            component Foo {
385                provides Hello h;
386            }
387
388            component Bar {
389                control;
390                uses Hello w;
391            }
392
393            assembly {
394                composition {
395                    component Foo f;
396                    component Bar b;
397                    connection Conn conn(from Foo.h, to Bar.w);
398                }
399            }
400            ''')
401
402    def test_multiline_comment_preserves_line_numbers(self):
403        _, content, _ = self.parser.parse_string('''
404            /* multiline comment
405             * that may affect line numbering
406             * if we've messed up
407             */
408            procedure P {}
409            ''')
410
411        self.assertEqual(content.head, 'start')
412        self.assertLen(content.tail, 1)
413        P = content.tail[0]
414        self.assertEqual(P.head, 'procedure_decl')
415
416        P.calc_position()
417        self.assertEqual(P.min_line, 6)
418
419    def test_manual_line_directives(self):
420        '''
421        GCC supports two forms of line directive:
422
423            #line <linenum> <filename>
424
425            # <linenum> <filename>
426
427        Only the first of these is documented in [0], though it is actually the
428        second of these that the C pre processor outputs. CAmkES' understanding
429        of these directives was previously based on observed directives from
430        CPP and it could not understand the first of these. This test checks
431        that it can now understand the first of these as well, as these may be
432        in use by some other non-CPP pre processing tool.
433
434          [0]: https://gcc.gnu.org/onlinedocs/cpp/Line-Control.html
435        '''
436
437        # Deliberately parse a malformed spec to generate an error.
438        try:
439            self.parser.parse_string('''
440
441                #line 42 "A"
442
443                connector Foo {
444                    from Procedure bar;
445                    to Procedure baz;
446                }
447                ''')
448
449            # If we reached this point, the malformed spec did not trigger an
450            # error as expected.
451            self.fail('incorrect syntax accepted by stage 1 parser')
452
453        except ParseError as e:
454            error = e
455
456        self.assertRegexpMatches(str(error), 'A:44:', 'alternate form of line '
457            'directive not supported')
458
459    def test_multiple_error_message_line_numbers(self):
460        '''
461        This test checks that when multiple PlyPlus errors are triggered on a
462        spec that has C pre-processor line directives, the correct source line
463        is still located in the error message. There was previously an issue
464        where line number information would not be provided in this case.
465        '''
466
467        # Parse a malformed spec with some line directives. Note that the spec
468        # contains the old form of connector definition.
469        try:
470            self.parser.parse_string('''
471
472                # 42 "A"
473
474                connector Foo {
475                    from Procedure bar;
476                    to Procedure baz;
477                }
478
479                connector Qux {
480                    from Procedure Moo;
481                    from Procedure Cow;
482                }
483                ''')
484
485            # If we reached this point, the malformed spec did not trigger an
486            # error as expected.
487            self.fail('incorrect syntax accepted by stage 1 parser')
488
489        except ParseError as e:
490            self.assertGreaterEqual(len(e.args), 2, 'only a '
491                'single error triggered when multiple were expected')
492
493            # If the line number narrowing algorithm has correctly taken the
494            # line directive into account, we should get the following prefix.
495            self.assertRegexpMatches(str(e), 'A:44:', 'line directive not '
496                'accounted for')
497
498    def test_list_dict_key(self):
499        '''
500        Test using a list as a key in a dictionary used as a setting. Should
501        not be accepted.
502        '''
503        with self.assertRaises(ParseError):
504            self.parser.parse_string(
505                'configuration {\n'
506                '  foo.bar = {[1, 2, 3]: 3};\n'
507                '}')
508
509    def test_c_include(self):
510        '''
511        Test we can parse a C relative path include.
512        '''
513        _, content, _ = self.parser.parse_string(
514            'procedure Foo { include "hello.h"; }')
515
516        self.assertEqual(content.head, 'start')
517        self.assertLen(content.tail, 1)
518
519        self.assertEqual(content.tail[0].head, 'procedure_decl')
520        proc_decl = content.tail[0]
521
522        self.assertLen(proc_decl.tail, 2)
523        self.assertEqual(proc_decl.tail[0].head, 'id')
524        self.assertLen(proc_decl.tail[0].tail, 1)
525        self.assertEqual(proc_decl.tail[0].tail[0], 'Foo')
526
527        self.assertEqual(proc_decl.tail[1].head, 'procedure_defn')
528        proc_defn = proc_decl.tail[1]
529
530        self.assertLen(proc_defn.tail, 1)
531        self.assertEqual(proc_defn.tail[0].head, 'include')
532        include = proc_defn.tail[0]
533
534        self.assertLen(include.tail, 1)
535        self.assertEqual(include.tail[0].head, 'multi_string')
536        multi_string = include.tail[0]
537
538        self.assertLen(multi_string.tail, 1)
539        self.assertEqual(multi_string.tail[0].head, 'quoted_string')
540        quoted_string = multi_string.tail[0]
541
542        self.assertLen(quoted_string.tail, 1)
543        self.assertEqual(quoted_string.tail[0], '"hello.h"')
544
545if __name__ == '__main__':
546    unittest.main()
547