Skip to content

Commit ad4018f

Browse files
tausbntausbn
committed
Python: Add parser support for lazy imports
As defined in PEP-810. We implement this in much the same way as how we handle `async` annotations currently. The relevant nodes get an `is_lazy` field that defaults to being false.
1 parent 6078df5 commit ad4018f

File tree

7 files changed

+337
-7
lines changed

7 files changed

+337
-7
lines changed

‎python/extractor/semmle/python/ast.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -845,17 +845,19 @@ def __init__(self, test, body, orelse):
845845

846846

847847
class Import(stmt):
848-
__slots__ = "names",
848+
__slots__ = "is_lazy", "names",
849849

850-
def __init__(self, names):
850+
def __init__(self, names, is_lazy=False):
851851
self.names = names
852+
self.is_lazy = is_lazy
852853

853854

854855
class ImportFrom(stmt):
855-
__slots__ = "module",
856+
__slots__ = "is_lazy", "module",
856857

857-
def __init__(self, module):
858+
def __init__(self, module, is_lazy=False):
858859
self.module = module
860+
self.is_lazy = is_lazy
859861

860862

861863
class Nonlocal(stmt):

‎python/extractor/semmle/python/parser/dump_ast.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,8 @@ def visit(self, node, level=0, visited=None):
7272
# just not print it in that case.
7373
if field == "parenthesised" and value is None:
7474
continue
75-
# Likewise, the default value for `is_async` is `False`, so we don't need to print it.
76-
if field == "is_async" and value is False:
75+
# Likewise, the default value for `is_async` and `is_lazy` is `False`, so we don't need to print it.
76+
if field in ("is_async", "is_lazy") and value is False:
7777
continue
7878
output.write("{} {}:".format(indent,field))
7979
if isinstance(value, list):

‎python/extractor/semmle/python/parser/tsg_parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ def create_placeholder_args(cls):
291291
if cls in (ast.Raise, ast.Ellipsis):
292292
return {}
293293
fields = ast_fields[cls]
294-
args = {field: None for field in fields if field != "is_async"}
294+
args = {field: None for field in fields if field not in ("is_async", "is_lazy")}
295295
for field in list_fields.get(cls, ()):
296296
args[field] = []
297297
if cls in (ast.GeneratorExp, ast.ListComp, ast.SetComp, ast.DictComp):

‎python/extractor/tests/parser/lazy_imports_new.expected‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
Module: [2, 0] - [35, 0]
2+
body: [
3+
Import: [2, 0] - [2, 13]
4+
is_lazy: True
5+
names: [
6+
alias: [2, 12] - [2, 13]
7+
value:
8+
ImportExpr: [2, 12] - [2, 13]
9+
level: 0
10+
name: 'a'
11+
top: True
12+
asname:
13+
Name: [2, 12] - [2, 13]
14+
variable: Variable('a', None)
15+
ctx: Store
16+
]
17+
Import: [4, 0] - [4, 18]
18+
is_lazy: True
19+
names: [
20+
alias: [4, 12] - [4, 14]
21+
value:
22+
ImportExpr: [4, 12] - [4, 14]
23+
level: 0
24+
name: 'b1'
25+
top: True
26+
asname:
27+
Name: [4, 12] - [4, 14]
28+
variable: Variable('b1', None)
29+
ctx: Store
30+
alias: [4, 16] - [4, 18]
31+
value:
32+
ImportExpr: [4, 16] - [4, 18]
33+
level: 0
34+
name: 'b2'
35+
top: True
36+
asname:
37+
Name: [4, 16] - [4, 18]
38+
variable: Variable('b2', None)
39+
ctx: Store
40+
]
41+
Import: [6, 0] - [6, 20]
42+
is_lazy: True
43+
names: [
44+
alias: [6, 12] - [6, 20]
45+
value:
46+
ImportExpr: [6, 12] - [6, 20]
47+
level: 0
48+
name: 'c1.c2.c3'
49+
top: True
50+
asname:
51+
Name: [6, 12] - [6, 20]
52+
variable: Variable('c1', None)
53+
ctx: Store
54+
]
55+
Import: [8, 0] - [8, 23]
56+
is_lazy: True
57+
names: [
58+
alias: [8, 12] - [8, 23]
59+
value:
60+
ImportExpr: [8, 12] - [8, 17]
61+
level: 0
62+
name: 'd1.d2'
63+
top: False
64+
asname:
65+
Name: [8, 21] - [8, 23]
66+
variable: Variable('d3', None)
67+
ctx: Store
68+
]
69+
Import: [10, 0] - [10, 20]
70+
is_lazy: True
71+
names: [
72+
alias: [10, 19] - [10, 20]
73+
value:
74+
ImportMember: [10, 19] - [10, 20]
75+
module:
76+
ImportExpr: [10, 10] - [10, 11]
77+
level: 0
78+
name: 'e'
79+
top: False
80+
name: 'f'
81+
asname:
82+
Name: [10, 19] - [10, 20]
83+
variable: Variable('f', None)
84+
ctx: Store
85+
]
86+
Import: [12, 0] - [12, 29]
87+
is_lazy: True
88+
names: [
89+
alias: [12, 23] - [12, 25]
90+
value:
91+
ImportMember: [12, 23] - [12, 25]
92+
module:
93+
ImportExpr: [12, 10] - [12, 15]
94+
level: 0
95+
name: 'g1.g2'
96+
top: False
97+
name: 'h1'
98+
asname:
99+
Name: [12, 23] - [12, 25]
100+
variable: Variable('h1', None)
101+
ctx: Store
102+
alias: [12, 27] - [12, 29]
103+
value:
104+
ImportMember: [12, 27] - [12, 29]
105+
module:
106+
ImportExpr: [12, 10] - [12, 15]
107+
level: 0
108+
name: 'g1.g2'
109+
top: False
110+
name: 'h2'
111+
asname:
112+
Name: [12, 27] - [12, 29]
113+
variable: Variable('h2', None)
114+
ctx: Store
115+
]
116+
Import: [14, 0] - [14, 32]
117+
is_lazy: True
118+
names: [
119+
alias: [14, 20] - [14, 28]
120+
value:
121+
ImportMember: [14, 20] - [14, 28]
122+
module:
123+
ImportExpr: [14, 10] - [14, 12]
124+
level: 0
125+
name: 'i1'
126+
top: False
127+
name: 'j1'
128+
asname:
129+
Name: [14, 26] - [14, 28]
130+
variable: Variable('j2', None)
131+
ctx: Store
132+
alias: [14, 30] - [14, 32]
133+
value:
134+
ImportMember: [14, 30] - [14, 32]
135+
module:
136+
ImportExpr: [14, 10] - [14, 12]
137+
level: 0
138+
name: 'i1'
139+
top: False
140+
name: 'j3'
141+
asname:
142+
Name: [14, 30] - [14, 32]
143+
variable: Variable('j3', None)
144+
ctx: Store
145+
]
146+
Import: [16, 0] - [16, 37]
147+
is_lazy: True
148+
names: [
149+
alias: [16, 25] - [16, 33]
150+
value:
151+
ImportMember: [16, 25] - [16, 33]
152+
module:
153+
ImportExpr: [16, 10] - [16, 17]
154+
level: 2
155+
name: 'k1.k2'
156+
top: False
157+
name: 'l1'
158+
asname:
159+
Name: [16, 31] - [16, 33]
160+
variable: Variable('l2', None)
161+
ctx: Store
162+
alias: [16, 35] - [16, 37]
163+
value:
164+
ImportMember: [16, 35] - [16, 37]
165+
module:
166+
ImportExpr: [16, 10] - [16, 17]
167+
level: 2
168+
name: 'k1.k2'
169+
top: False
170+
name: 'l3'
171+
asname:
172+
Name: [16, 35] - [16, 37]
173+
variable: Variable('l3', None)
174+
ctx: Store
175+
]
176+
Import: [18, 0] - [18, 20]
177+
is_lazy: True
178+
names: [
179+
alias: [18, 19] - [18, 20]
180+
value:
181+
ImportMember: [18, 19] - [18, 20]
182+
module:
183+
ImportExpr: [18, 10] - [18, 11]
184+
level: 1
185+
name: None
186+
top: False
187+
name: 'm'
188+
asname:
189+
Name: [18, 19] - [18, 20]
190+
variable: Variable('m', None)
191+
ctx: Store
192+
]
193+
Import: [20, 0] - [20, 22]
194+
is_lazy: True
195+
names: [
196+
alias: [20, 21] - [20, 22]
197+
value:
198+
ImportMember: [20, 21] - [20, 22]
199+
module:
200+
ImportExpr: [20, 10] - [20, 13]
201+
level: 3
202+
name: None
203+
top: False
204+
name: 'n'
205+
asname:
206+
Name: [20, 21] - [20, 22]
207+
variable: Variable('n', None)
208+
ctx: Store
209+
]
210+
ImportFrom: [22, 0] - [22, 20]
211+
is_lazy: True
212+
module:
213+
ImportExpr: [22, 10] - [22, 11]
214+
level: 0
215+
name: 'o'
216+
top: False
217+
Assign: [26, 0] - [26, 8]
218+
targets: [
219+
Name: [26, 0] - [26, 4]
220+
variable: Variable('lazy', None)
221+
ctx: Store
222+
]
223+
value:
224+
Num: [26, 7] - [26, 8]
225+
n: 1
226+
text: '1'
227+
Assign: [28, 0] - [28, 11]
228+
targets: [
229+
Subscript: [28, 0] - [28, 7]
230+
value:
231+
Name: [28, 0] - [28, 4]
232+
variable: Variable('lazy', None)
233+
ctx: Load
234+
index:
235+
Num: [28, 5] - [28, 6]
236+
n: 2
237+
text: '2'
238+
ctx: Store
239+
]
240+
value:
241+
Num: [28, 10] - [28, 11]
242+
n: 3
243+
text: '3'
244+
Assign: [30, 0] - [30, 12]
245+
targets: [
246+
Attribute: [30, 0] - [30, 8]
247+
value:
248+
Name: [30, 0] - [30, 4]
249+
variable: Variable('lazy', None)
250+
ctx: Load
251+
attr: 'foo'
252+
ctx: Store
253+
]
254+
value:
255+
Num: [30, 11] - [30, 12]
256+
n: 4
257+
text: '4'
258+
Expr: [32, 0] - [32, 6]
259+
value:
260+
Call: [32, 0] - [32, 6]
261+
func:
262+
Name: [32, 0] - [32, 4]
263+
variable: Variable('lazy', None)
264+
ctx: Load
265+
positional_args: []
266+
named_args: []
267+
AnnAssign: [34, 0] - [34, 14]
268+
value: None
269+
annotation:
270+
Name: [34, 10] - [34, 14]
271+
variable: Variable('case', None)
272+
ctx: Load
273+
target:
274+
Subscript: [34, 0] - [34, 7]
275+
value:
276+
Name: [34, 0] - [34, 4]
277+
variable: Variable('lazy', None)
278+
ctx: Load
279+
index:
280+
Num: [34, 5] - [34, 6]
281+
n: 5
282+
text: '5'
283+
ctx: Store
284+
]

‎python/extractor/tests/parser/lazy_imports_new.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Basic lazy imports (PEP 810)
2+
lazy import a
3+
4+
lazy import b1, b2
5+
6+
lazy import c1.c2.c3
7+
8+
lazy import d1.d2 as d3
9+
10+
lazy from e import f
11+
12+
lazy from g1.g2 import h1, h2
13+
14+
lazy from i1 import j1 as j2, j3
15+
16+
lazy from ..k1.k2 import l1 as l2, l3
17+
18+
lazy from . import m
19+
20+
lazy from ... import n
21+
22+
lazy from o import *
23+
24+
25+
# `lazy` used as a regular identifier (soft keyword behavior)
26+
lazy = 1
27+
28+
lazy[2] = 3
29+
30+
lazy.foo = 4
31+
32+
lazy()
33+
34+
lazy[5] : case

‎python/extractor/tsg-python/python.tsg‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1777,6 +1777,13 @@
17771777

17781778
attr (@importfrom.importexpr) level = level
17791779
}
1780+
; Set is_lazy for lazy import statements (PEP 810)
1781+
[
1782+
(import_statement is_lazy: _)
1783+
(import_from_statement is_lazy: _)
1784+
] @lazy_import
1785+
{ attr (@lazy_import.node) is_lazy = #true }
1786+
17801787
;;;;;; End of Import (`from ... import ...`)
17811788

17821789
;;;;;; Raise (`raise ...`)

0 commit comments

Comments
 (0)