Complete Yocto mirror with license table for TQMa6UL (2038-compliance)
- 264 license table entries with exact download URLs (224/264 resolved) - Complete sources/ directory with all BitBake recipes - Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl) - Full traceability for Softwarefreigabeantrag - GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4 - License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)
This commit is contained in:
Siggi (OpenClaw Agent)
0
sources/poky/bitbake/lib/bb/pysh/__init__.py
Normal file
0
sources/poky/bitbake/lib/bb/pysh/__init__.py
Normal file
883
sources/poky/bitbake/lib/bb/pysh/pyshlex.py
Normal file
883
sources/poky/bitbake/lib/bb/pysh/pyshlex.py
Normal file
@@ -0,0 +1,883 @@
|
||||
# pyshlex.py - PLY compatible lexer for pysh.
|
||||
#
|
||||
# Copyright 2007 Patrick Mezard
|
||||
#
|
||||
# This software may be used and distributed according to the terms
|
||||
# of the GNU General Public License, incorporated herein by reference.
|
||||
|
||||
# TODO:
|
||||
# - review all "char in 'abc'" snippets: the empty string can be matched
|
||||
# - test line continuations within quoted/expansion strings
|
||||
# - eof is buggy wrt sublexers
|
||||
# - the lexer cannot really work in pull mode as it would be required to run
|
||||
# PLY in pull mode. It was designed to work incrementally and it would not be
|
||||
# that hard to enable pull mode.
|
||||
import re
|
||||
|
||||
from ply import lex
|
||||
from bb.pysh.sherrors import *
|
||||
|
||||
class NeedMore(Exception):
|
||||
pass
|
||||
|
||||
def is_blank(c):
|
||||
return c in (' ', '\t')
|
||||
|
||||
_RE_DIGITS = re.compile(r'^\d+$')
|
||||
|
||||
def are_digits(s):
|
||||
return _RE_DIGITS.search(s) is not None
|
||||
|
||||
_OPERATORS = dict([
|
||||
('&&', 'AND_IF'),
|
||||
('||', 'OR_IF'),
|
||||
(';;', 'DSEMI'),
|
||||
('<<', 'DLESS'),
|
||||
('>>', 'DGREAT'),
|
||||
('<&', 'LESSAND'),
|
||||
('>&', 'GREATAND'),
|
||||
('<>', 'LESSGREAT'),
|
||||
('<<-', 'DLESSDASH'),
|
||||
('>|', 'CLOBBER'),
|
||||
('&', 'AMP'),
|
||||
(';', 'COMMA'),
|
||||
('<', 'LESS'),
|
||||
('>', 'GREATER'),
|
||||
('(', 'LPARENS'),
|
||||
(')', 'RPARENS'),
|
||||
])
|
||||
|
||||
#Make a function to silence pychecker "Local variable shadows global"
|
||||
def make_partial_ops():
|
||||
partials = {}
|
||||
for k in _OPERATORS:
|
||||
for i in range(1, len(k)+1):
|
||||
partials[k[:i]] = None
|
||||
return partials
|
||||
|
||||
_PARTIAL_OPERATORS = make_partial_ops()
|
||||
|
||||
def is_partial_op(s):
|
||||
"""Return True if s matches a non-empty subpart of an operator starting
|
||||
at its first character.
|
||||
"""
|
||||
return s in _PARTIAL_OPERATORS
|
||||
|
||||
def is_op(s):
|
||||
"""If s matches an operator, returns the operator identifier. Return None
|
||||
otherwise.
|
||||
"""
|
||||
return _OPERATORS.get(s)
|
||||
|
||||
_RESERVEDS = dict([
|
||||
('if', 'If'),
|
||||
('then', 'Then'),
|
||||
('else', 'Else'),
|
||||
('elif', 'Elif'),
|
||||
('fi', 'Fi'),
|
||||
('do', 'Do'),
|
||||
('done', 'Done'),
|
||||
('case', 'Case'),
|
||||
('esac', 'Esac'),
|
||||
('while', 'While'),
|
||||
('until', 'Until'),
|
||||
('for', 'For'),
|
||||
('{', 'Lbrace'),
|
||||
('}', 'Rbrace'),
|
||||
('!', 'Bang'),
|
||||
('in', 'In'),
|
||||
('|', 'PIPE'),
|
||||
])
|
||||
|
||||
def get_reserved(s):
|
||||
return _RESERVEDS.get(s)
|
||||
|
||||
_RE_NAME = re.compile(r'^[0-9a-zA-Z_]+$')
|
||||
|
||||
def is_name(s):
|
||||
return _RE_NAME.search(s) is not None
|
||||
|
||||
def find_chars(seq, chars):
|
||||
for i,v in enumerate(seq):
|
||||
if v in chars:
|
||||
return i,v
|
||||
return -1, None
|
||||
|
||||
class WordLexer:
|
||||
"""WordLexer parse quoted or expansion expressions and return an expression
|
||||
tree. The input string can be any well formed sequence beginning with quoting
|
||||
or expansion character. Embedded expressions are handled recursively. The
|
||||
resulting tree is made of lists and strings. Lists represent quoted or
|
||||
expansion expressions. Each list first element is the opening separator,
|
||||
the last one the closing separator. In-between can be any number of strings
|
||||
or lists for sub-expressions. Non quoted/expansion expression can written as
|
||||
strings or as lists with empty strings as starting and ending delimiters.
|
||||
"""
|
||||
|
||||
NAME_CHARSET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
|
||||
NAME_CHARSET = dict(zip(NAME_CHARSET, NAME_CHARSET))
|
||||
|
||||
SPECIAL_CHARSET = '@*#?-$!0'
|
||||
|
||||
#Characters which can be escaped depends on the current delimiters
|
||||
ESCAPABLE = {
|
||||
'`': set(['$', '\\', '`']),
|
||||
'"': set(['$', '\\', '`', '"']),
|
||||
"'": set(),
|
||||
}
|
||||
|
||||
def __init__(self, heredoc = False):
|
||||
# _buffer is the unprocessed input characters buffer
|
||||
self._buffer = []
|
||||
# _stack is empty or contains a quoted list being processed
|
||||
# (this is the DFS path to the quoted expression being evaluated).
|
||||
self._stack = []
|
||||
self._escapable = None
|
||||
# True when parsing unquoted here documents
|
||||
self._heredoc = heredoc
|
||||
|
||||
def add(self, data, eof=False):
|
||||
"""Feed the lexer with more data. If the quoted expression can be
|
||||
delimited, return a tuple (expr, remaining) containing the expression
|
||||
tree and the unconsumed data.
|
||||
Otherwise, raise NeedMore.
|
||||
"""
|
||||
self._buffer += list(data)
|
||||
self._parse(eof)
|
||||
|
||||
result = self._stack[0]
|
||||
remaining = ''.join(self._buffer)
|
||||
self._stack = []
|
||||
self._buffer = []
|
||||
return result, remaining
|
||||
|
||||
def _is_escapable(self, c, delim=None):
|
||||
if delim is None:
|
||||
if self._heredoc:
|
||||
# Backslashes works as if they were double quoted in unquoted
|
||||
# here-documents
|
||||
delim = '"'
|
||||
else:
|
||||
if len(self._stack)<=1:
|
||||
return True
|
||||
delim = self._stack[-2][0]
|
||||
|
||||
escapables = self.ESCAPABLE.get(delim, None)
|
||||
return escapables is None or c in escapables
|
||||
|
||||
def _parse_squote(self, buf, result, eof):
|
||||
if not buf:
|
||||
raise NeedMore()
|
||||
try:
|
||||
pos = buf.index("'")
|
||||
except ValueError:
|
||||
raise NeedMore()
|
||||
result[-1] += ''.join(buf[:pos])
|
||||
result += ["'"]
|
||||
return pos+1, True
|
||||
|
||||
def _parse_bquote(self, buf, result, eof):
|
||||
if not buf:
|
||||
raise NeedMore()
|
||||
|
||||
if buf[0]=='\n':
|
||||
#Remove line continuations
|
||||
result[:] = ['', '', '']
|
||||
elif self._is_escapable(buf[0]):
|
||||
result[-1] += buf[0]
|
||||
result += ['']
|
||||
else:
|
||||
#Keep as such
|
||||
result[:] = ['', '\\'+buf[0], '']
|
||||
|
||||
return 1, True
|
||||
|
||||
def _parse_dquote(self, buf, result, eof):
|
||||
if not buf:
|
||||
raise NeedMore()
|
||||
pos, sep = find_chars(buf, '$\\`"')
|
||||
if pos==-1:
|
||||
raise NeedMore()
|
||||
|
||||
result[-1] += ''.join(buf[:pos])
|
||||
if sep=='"':
|
||||
result += ['"']
|
||||
return pos+1, True
|
||||
else:
|
||||
#Keep everything until the separator and defer processing
|
||||
return pos, False
|
||||
|
||||
def _parse_command(self, buf, result, eof):
|
||||
if not buf:
|
||||
raise NeedMore()
|
||||
|
||||
chars = '$\\`"\''
|
||||
if result[0] == '$(':
|
||||
chars += ')'
|
||||
pos, sep = find_chars(buf, chars)
|
||||
if pos == -1:
|
||||
raise NeedMore()
|
||||
|
||||
result[-1] += ''.join(buf[:pos])
|
||||
if (result[0]=='$(' and sep==')') or (result[0]=='`' and sep=='`'):
|
||||
result += [sep]
|
||||
return pos+1, True
|
||||
else:
|
||||
return pos, False
|
||||
|
||||
def _parse_parameter(self, buf, result, eof):
|
||||
if not buf:
|
||||
raise NeedMore()
|
||||
|
||||
pos, sep = find_chars(buf, '$\\`"\'}')
|
||||
if pos==-1:
|
||||
raise NeedMore()
|
||||
|
||||
result[-1] += ''.join(buf[:pos])
|
||||
if sep=='}':
|
||||
result += [sep]
|
||||
return pos+1, True
|
||||
else:
|
||||
return pos, False
|
||||
|
||||
def _parse_dollar(self, buf, result, eof):
|
||||
sep = result[0]
|
||||
if sep=='$':
|
||||
if not buf:
|
||||
#TODO: handle empty $
|
||||
raise NeedMore()
|
||||
if buf[0]=='(':
|
||||
if len(buf)==1:
|
||||
raise NeedMore()
|
||||
|
||||
if buf[1]=='(':
|
||||
result[0] = '$(('
|
||||
buf[:2] = []
|
||||
else:
|
||||
result[0] = '$('
|
||||
buf[:1] = []
|
||||
|
||||
elif buf[0]=='{':
|
||||
result[0] = '${'
|
||||
buf[:1] = []
|
||||
else:
|
||||
if buf[0] in self.SPECIAL_CHARSET:
|
||||
result[-1] = buf[0]
|
||||
read = 1
|
||||
else:
|
||||
for read,c in enumerate(buf):
|
||||
if c not in self.NAME_CHARSET:
|
||||
break
|
||||
else:
|
||||
if not eof:
|
||||
raise NeedMore()
|
||||
read += 1
|
||||
|
||||
result[-1] += ''.join(buf[0:read])
|
||||
|
||||
if not result[-1]:
|
||||
result[:] = ['', result[0], '']
|
||||
else:
|
||||
result += ['']
|
||||
return read,True
|
||||
|
||||
sep = result[0]
|
||||
if sep=='$(':
|
||||
parsefunc = self._parse_command
|
||||
elif sep=='${':
|
||||
parsefunc = self._parse_parameter
|
||||
else:
|
||||
raise NotImplementedError(sep)
|
||||
|
||||
pos, closed = parsefunc(buf, result, eof)
|
||||
return pos, closed
|
||||
|
||||
def _parse(self, eof):
|
||||
buf = self._buffer
|
||||
stack = self._stack
|
||||
recurse = False
|
||||
|
||||
while 1:
|
||||
if not stack or recurse:
|
||||
if not buf:
|
||||
raise NeedMore()
|
||||
if buf[0] not in ('"\\`$\''):
|
||||
raise ShellSyntaxError('Invalid quoted string sequence')
|
||||
stack.append([buf[0], ''])
|
||||
buf[:1] = []
|
||||
recurse = False
|
||||
|
||||
result = stack[-1]
|
||||
if result[0]=="'":
|
||||
parsefunc = self._parse_squote
|
||||
elif result[0]=='\\':
|
||||
parsefunc = self._parse_bquote
|
||||
elif result[0]=='"':
|
||||
parsefunc = self._parse_dquote
|
||||
elif result[0]=='`':
|
||||
parsefunc = self._parse_command
|
||||
elif result[0][0]=='$':
|
||||
parsefunc = self._parse_dollar
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
read, closed = parsefunc(buf, result, eof)
|
||||
|
||||
buf[:read] = []
|
||||
if closed:
|
||||
if len(stack)>1:
|
||||
#Merge in parent expression
|
||||
parsed = stack.pop()
|
||||
stack[-1] += [parsed]
|
||||
stack[-1] += ['']
|
||||
else:
|
||||
break
|
||||
else:
|
||||
recurse = True
|
||||
|
||||
def normalize_wordtree(wtree):
|
||||
"""Fold back every literal sequence (delimited with empty strings) into
|
||||
parent sequence.
|
||||
"""
|
||||
def normalize(wtree):
|
||||
result = []
|
||||
for part in wtree[1:-1]:
|
||||
if isinstance(part, list):
|
||||
part = normalize(part)
|
||||
if part[0]=='':
|
||||
#Move the part content back at current level
|
||||
result += part[1:-1]
|
||||
continue
|
||||
elif not part:
|
||||
#Remove empty strings
|
||||
continue
|
||||
result.append(part)
|
||||
if not result:
|
||||
result = ['']
|
||||
return [wtree[0]] + result + [wtree[-1]]
|
||||
|
||||
return normalize(wtree)
|
||||
|
||||
|
||||
def make_wordtree(token, here_document=False):
|
||||
"""Parse a delimited token and return a tree similar to the ones returned by
|
||||
WordLexer. token may contain any combinations of expansion/quoted fields and
|
||||
non-ones.
|
||||
"""
|
||||
tree = ['']
|
||||
remaining = token
|
||||
delimiters = '\\$`'
|
||||
if not here_document:
|
||||
delimiters += '\'"'
|
||||
|
||||
while 1:
|
||||
pos, sep = find_chars(remaining, delimiters)
|
||||
if pos==-1:
|
||||
tree += [remaining, '']
|
||||
return normalize_wordtree(tree)
|
||||
tree.append(remaining[:pos])
|
||||
remaining = remaining[pos:]
|
||||
|
||||
try:
|
||||
result, remaining = WordLexer(heredoc = here_document).add(remaining, True)
|
||||
except NeedMore:
|
||||
raise ShellSyntaxError('Invalid token "%s"')
|
||||
tree.append(result)
|
||||
|
||||
|
||||
def wordtree_as_string(wtree):
|
||||
"""Rewrite an expression tree generated by make_wordtree as string."""
|
||||
def visit(node, output):
|
||||
for child in node:
|
||||
if isinstance(child, list):
|
||||
visit(child, output)
|
||||
else:
|
||||
output.append(child)
|
||||
|
||||
output = []
|
||||
visit(wtree, output)
|
||||
return ''.join(output)
|
||||
|
||||
|
||||
def unquote_wordtree(wtree):
|
||||
"""Fold the word tree while removing quotes everywhere. Other expansion
|
||||
sequences are joined as such.
|
||||
"""
|
||||
def unquote(wtree):
|
||||
unquoted = []
|
||||
if wtree[0] in ('', "'", '"', '\\'):
|
||||
wtree = wtree[1:-1]
|
||||
|
||||
for part in wtree:
|
||||
if isinstance(part, list):
|
||||
part = unquote(part)
|
||||
unquoted.append(part)
|
||||
return ''.join(unquoted)
|
||||
|
||||
return unquote(wtree)
|
||||
|
||||
|
||||
class HereDocLexer:
|
||||
"""HereDocLexer delimits whatever comes from the here-document starting newline
|
||||
not included to the closing delimiter line included.
|
||||
"""
|
||||
def __init__(self, op, delim):
|
||||
assert op in ('<<', '<<-')
|
||||
if not delim:
|
||||
raise ShellSyntaxError('invalid here document delimiter %s' % str(delim))
|
||||
|
||||
self._op = op
|
||||
self._delim = delim
|
||||
self._buffer = []
|
||||
self._token = []
|
||||
|
||||
def add(self, data, eof):
|
||||
"""If the here-document was delimited, return a tuple (content, remaining).
|
||||
Raise NeedMore() otherwise.
|
||||
"""
|
||||
self._buffer += list(data)
|
||||
self._parse(eof)
|
||||
token = ''.join(self._token)
|
||||
remaining = ''.join(self._buffer)
|
||||
self._token, self._remaining = [], []
|
||||
return token, remaining
|
||||
|
||||
def _parse(self, eof):
|
||||
while 1:
|
||||
#Look for first unescaped newline. Quotes may be ignored
|
||||
escaped = False
|
||||
for i,c in enumerate(self._buffer):
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif c=='\\':
|
||||
escaped = True
|
||||
elif c=='\n':
|
||||
break
|
||||
else:
|
||||
i = -1
|
||||
|
||||
if i==-1 or self._buffer[i]!='\n':
|
||||
if not eof:
|
||||
raise NeedMore()
|
||||
#No more data, maybe the last line is closing delimiter
|
||||
line = ''.join(self._buffer)
|
||||
eol = ''
|
||||
self._buffer[:] = []
|
||||
else:
|
||||
line = ''.join(self._buffer[:i])
|
||||
eol = self._buffer[i]
|
||||
self._buffer[:i+1] = []
|
||||
|
||||
if self._op=='<<-':
|
||||
line = line.lstrip('\t')
|
||||
|
||||
if line==self._delim:
|
||||
break
|
||||
|
||||
self._token += [line, eol]
|
||||
if i==-1:
|
||||
break
|
||||
|
||||
class Token:
|
||||
#TODO: check this is still in use
|
||||
OPERATOR = 'OPERATOR'
|
||||
WORD = 'WORD'
|
||||
|
||||
def __init__(self):
|
||||
self.value = ''
|
||||
self.type = None
|
||||
|
||||
def __getitem__(self, key):
|
||||
#Behave like a two elements tuple
|
||||
if key==0:
|
||||
return self.type
|
||||
if key==1:
|
||||
return self.value
|
||||
raise IndexError(key)
|
||||
|
||||
|
||||
class HereDoc:
|
||||
def __init__(self, op, name=None):
|
||||
self.op = op
|
||||
self.name = name
|
||||
self.pendings = []
|
||||
|
||||
TK_COMMA = 'COMMA'
|
||||
TK_AMPERSAND = 'AMP'
|
||||
TK_OP = 'OP'
|
||||
TK_TOKEN = 'TOKEN'
|
||||
TK_COMMENT = 'COMMENT'
|
||||
TK_NEWLINE = 'NEWLINE'
|
||||
TK_IONUMBER = 'IO_NUMBER'
|
||||
TK_ASSIGNMENT = 'ASSIGNMENT_WORD'
|
||||
TK_HERENAME = 'HERENAME'
|
||||
|
||||
class Lexer:
|
||||
"""Main lexer.
|
||||
|
||||
Call add() until the script AST is returned.
|
||||
"""
|
||||
# Here-document handling makes the whole thing more complex because they basically
|
||||
# force tokens to be reordered: here-content must come right after the operator
|
||||
# and the here-document name, while some other tokens might be following the
|
||||
# here-document expression on the same line.
|
||||
#
|
||||
# So, here-doc states are basically:
|
||||
# *self._state==ST_NORMAL
|
||||
# - self._heredoc.op is None: no here-document
|
||||
# - self._heredoc.op is not None but name is: here-document operator matched,
|
||||
# waiting for the document name/delimiter
|
||||
# - self._heredoc.op and name are not None: here-document is ready, following
|
||||
# tokens are being stored and will be pushed again when the document is
|
||||
# completely parsed.
|
||||
# *self._state==ST_HEREDOC
|
||||
# - The here-document is being delimited by self._herelexer. Once it is done
|
||||
# the content is pushed in front of the pending token list then all these
|
||||
# tokens are pushed once again.
|
||||
ST_NORMAL = 'ST_NORMAL'
|
||||
ST_OP = 'ST_OP'
|
||||
ST_BACKSLASH = 'ST_BACKSLASH'
|
||||
ST_QUOTED = 'ST_QUOTED'
|
||||
ST_COMMENT = 'ST_COMMENT'
|
||||
ST_HEREDOC = 'ST_HEREDOC'
|
||||
|
||||
#Match end of backquote strings
|
||||
RE_BACKQUOTE_END = re.compile(r'(?<!\\)(`)')
|
||||
|
||||
def __init__(self, parent_state = None):
|
||||
self._input = []
|
||||
self._pos = 0
|
||||
|
||||
self._token = ''
|
||||
self._type = TK_TOKEN
|
||||
|
||||
self._state = self.ST_NORMAL
|
||||
self._parent_state = parent_state
|
||||
self._wordlexer = None
|
||||
|
||||
self._heredoc = HereDoc(None)
|
||||
self._herelexer = None
|
||||
|
||||
### Following attributes are not used for delimiting token and can safely
|
||||
### be changed after here-document detection (see _push_toke)
|
||||
|
||||
# Count the number of tokens following a 'For' reserved word. Needed to
|
||||
# return an 'In' reserved word if it comes in third place.
|
||||
self._for_count = None
|
||||
|
||||
def add(self, data, eof=False):
|
||||
"""Feed the lexer with data.
|
||||
|
||||
When eof is set to True, returns unconsumed data or raise if the lexer
|
||||
is in the middle of a delimiting operation.
|
||||
Raise NeedMore otherwise.
|
||||
"""
|
||||
self._input += list(data)
|
||||
self._parse(eof)
|
||||
self._input[:self._pos] = []
|
||||
return ''.join(self._input)
|
||||
|
||||
def _parse(self, eof):
|
||||
while self._state:
|
||||
if self._pos>=len(self._input):
|
||||
if not eof:
|
||||
raise NeedMore()
|
||||
elif self._state not in (self.ST_OP, self.ST_QUOTED, self.ST_HEREDOC):
|
||||
#Delimit the current token and leave cleanly
|
||||
self._push_token('')
|
||||
break
|
||||
else:
|
||||
#Let the sublexer handle the eof themselves
|
||||
pass
|
||||
|
||||
if self._state==self.ST_NORMAL:
|
||||
self._parse_normal()
|
||||
elif self._state==self.ST_COMMENT:
|
||||
self._parse_comment()
|
||||
elif self._state==self.ST_OP:
|
||||
self._parse_op(eof)
|
||||
elif self._state==self.ST_QUOTED:
|
||||
self._parse_quoted(eof)
|
||||
elif self._state==self.ST_HEREDOC:
|
||||
self._parse_heredoc(eof)
|
||||
else:
|
||||
assert False, "Unknown state " + str(self._state)
|
||||
|
||||
if self._heredoc.op is not None:
|
||||
raise ShellSyntaxError('missing here-document delimiter')
|
||||
|
||||
def _parse_normal(self):
|
||||
c = self._input[self._pos]
|
||||
if c=='\n':
|
||||
self._push_token(c)
|
||||
self._token = c
|
||||
self._type = TK_NEWLINE
|
||||
self._push_token('')
|
||||
self._pos += 1
|
||||
elif c in ('\\', '\'', '"', '`', '$'):
|
||||
self._state = self.ST_QUOTED
|
||||
elif is_partial_op(c):
|
||||
self._push_token(c)
|
||||
|
||||
self._type = TK_OP
|
||||
self._token += c
|
||||
self._pos += 1
|
||||
self._state = self.ST_OP
|
||||
elif is_blank(c):
|
||||
self._push_token(c)
|
||||
|
||||
#Discard blanks
|
||||
self._pos += 1
|
||||
elif self._token:
|
||||
self._token += c
|
||||
self._pos += 1
|
||||
elif c=='#':
|
||||
self._state = self.ST_COMMENT
|
||||
self._type = TK_COMMENT
|
||||
self._pos += 1
|
||||
else:
|
||||
self._pos += 1
|
||||
self._token += c
|
||||
|
||||
def _parse_op(self, eof):
|
||||
assert self._token
|
||||
|
||||
while 1:
|
||||
if self._pos>=len(self._input):
|
||||
if not eof:
|
||||
raise NeedMore()
|
||||
c = ''
|
||||
else:
|
||||
c = self._input[self._pos]
|
||||
|
||||
op = self._token + c
|
||||
if c and is_partial_op(op):
|
||||
#Still parsing an operator
|
||||
self._token = op
|
||||
self._pos += 1
|
||||
else:
|
||||
#End of operator
|
||||
self._push_token(c)
|
||||
self._state = self.ST_NORMAL
|
||||
break
|
||||
|
||||
def _parse_comment(self):
|
||||
while 1:
|
||||
if self._pos>=len(self._input):
|
||||
raise NeedMore()
|
||||
|
||||
c = self._input[self._pos]
|
||||
if c=='\n':
|
||||
#End of comment, do not consume the end of line
|
||||
self._state = self.ST_NORMAL
|
||||
break
|
||||
else:
|
||||
self._token += c
|
||||
self._pos += 1
|
||||
|
||||
def _parse_quoted(self, eof):
|
||||
"""Precondition: the starting backquote/dollar is still in the input queue."""
|
||||
if not self._wordlexer:
|
||||
self._wordlexer = WordLexer()
|
||||
|
||||
if self._pos<len(self._input):
|
||||
#Transfer input queue character into the subparser
|
||||
input = self._input[self._pos:]
|
||||
self._pos += len(input)
|
||||
|
||||
wtree, remaining = self._wordlexer.add(input, eof)
|
||||
self._wordlexer = None
|
||||
self._token += wordtree_as_string(wtree)
|
||||
|
||||
#Put unparsed character back in the input queue
|
||||
if remaining:
|
||||
self._input[self._pos:self._pos] = list(remaining)
|
||||
self._state = self.ST_NORMAL
|
||||
|
||||
def _parse_heredoc(self, eof):
|
||||
assert not self._token
|
||||
|
||||
if self._herelexer is None:
|
||||
self._herelexer = HereDocLexer(self._heredoc.op, self._heredoc.name)
|
||||
|
||||
if self._pos<len(self._input):
|
||||
#Transfer input queue character into the subparser
|
||||
input = self._input[self._pos:]
|
||||
self._pos += len(input)
|
||||
|
||||
self._token, remaining = self._herelexer.add(input, eof)
|
||||
|
||||
#Reset here-document state
|
||||
self._herelexer = None
|
||||
heredoc, self._heredoc = self._heredoc, HereDoc(None)
|
||||
if remaining:
|
||||
self._input[self._pos:self._pos] = list(remaining)
|
||||
self._state = self.ST_NORMAL
|
||||
|
||||
#Push pending tokens
|
||||
heredoc.pendings[:0] = [(self._token, self._type, heredoc.name)]
|
||||
for token, type, delim in heredoc.pendings:
|
||||
self._token = token
|
||||
self._type = type
|
||||
self._push_token(delim)
|
||||
|
||||
def _push_token(self, delim):
|
||||
if not self._token:
|
||||
return 0
|
||||
|
||||
if self._heredoc.op is not None:
|
||||
if self._heredoc.name is None:
|
||||
#Here-document name
|
||||
if self._type!=TK_TOKEN:
|
||||
raise ShellSyntaxError("expecting here-document name, got '%s'" % self._token)
|
||||
self._heredoc.name = unquote_wordtree(make_wordtree(self._token))
|
||||
self._type = TK_HERENAME
|
||||
else:
|
||||
#Capture all tokens until the newline starting the here-document
|
||||
if self._type==TK_NEWLINE:
|
||||
assert self._state==self.ST_NORMAL
|
||||
self._state = self.ST_HEREDOC
|
||||
|
||||
self._heredoc.pendings.append((self._token, self._type, delim))
|
||||
self._token = ''
|
||||
self._type = TK_TOKEN
|
||||
return 1
|
||||
|
||||
# BEWARE: do not change parser state from here to the end of the function:
|
||||
# when parsing between an here-document operator to the end of the line
|
||||
# tokens are stored in self._heredoc.pendings. Therefore, they will not
|
||||
# reach the section below.
|
||||
|
||||
#Check operators
|
||||
if self._type==TK_OP:
|
||||
#False positive because of partial op matching
|
||||
op = is_op(self._token)
|
||||
if not op:
|
||||
self._type = TK_TOKEN
|
||||
else:
|
||||
#Map to the specific operator
|
||||
self._type = op
|
||||
if self._token in ('<<', '<<-'):
|
||||
#Done here rather than in _parse_op because there is no need
|
||||
#to change the parser state since we are still waiting for
|
||||
#the here-document name
|
||||
if self._heredoc.op is not None:
|
||||
raise ShellSyntaxError("syntax error near token '%s'" % self._token)
|
||||
assert self._heredoc.op is None
|
||||
self._heredoc.op = self._token
|
||||
|
||||
if self._type==TK_TOKEN:
|
||||
if '=' in self._token and not delim:
|
||||
if self._token.startswith('='):
|
||||
#Token is a WORD... a TOKEN that is.
|
||||
pass
|
||||
else:
|
||||
prev = self._token[:self._token.find('=')]
|
||||
if is_name(prev):
|
||||
self._type = TK_ASSIGNMENT
|
||||
else:
|
||||
#Just a token (unspecified)
|
||||
pass
|
||||
else:
|
||||
reserved = get_reserved(self._token)
|
||||
if reserved is not None:
|
||||
if reserved=='In' and self._for_count!=2:
|
||||
#Sorry, not a reserved word after all
|
||||
pass
|
||||
else:
|
||||
self._type = reserved
|
||||
if reserved in ('For', 'Case'):
|
||||
self._for_count = 0
|
||||
elif are_digits(self._token) and delim in ('<', '>'):
|
||||
#Detect IO_NUMBER
|
||||
self._type = TK_IONUMBER
|
||||
elif self._token==';':
|
||||
self._type = TK_COMMA
|
||||
elif self._token=='&':
|
||||
self._type = TK_AMPERSAND
|
||||
elif self._type==TK_COMMENT:
|
||||
#Comments are not part of sh grammar, ignore them
|
||||
self._token = ''
|
||||
self._type = TK_TOKEN
|
||||
return 0
|
||||
|
||||
if self._for_count is not None:
|
||||
#Track token count in 'For' expression to detect 'In' reserved words.
|
||||
#Can only be in third position, no need to go beyond
|
||||
self._for_count += 1
|
||||
if self._for_count==3:
|
||||
self._for_count = None
|
||||
|
||||
self.on_token((self._token, self._type))
|
||||
self._token = ''
|
||||
self._type = TK_TOKEN
|
||||
return 1
|
||||
|
||||
def on_token(self, token):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
tokens = [
|
||||
TK_TOKEN,
|
||||
# To silence yacc unused token warnings
|
||||
# TK_COMMENT,
|
||||
TK_NEWLINE,
|
||||
TK_IONUMBER,
|
||||
TK_ASSIGNMENT,
|
||||
TK_HERENAME,
|
||||
]
|
||||
|
||||
#Add specific operators
|
||||
tokens += _OPERATORS.values()
|
||||
#Add reserved words
|
||||
tokens += _RESERVEDS.values()
|
||||
|
||||
class PLYLexer(Lexer):
|
||||
"""Bridge Lexer and PLY lexer interface."""
|
||||
def __init__(self):
|
||||
Lexer.__init__(self)
|
||||
self._tokens = []
|
||||
self._current = 0
|
||||
self.lineno = 0
|
||||
|
||||
def on_token(self, token):
|
||||
value, type = token
|
||||
|
||||
self.lineno = 0
|
||||
t = lex.LexToken()
|
||||
t.value = value
|
||||
t.type = type
|
||||
t.lexer = self
|
||||
t.lexpos = 0
|
||||
t.lineno = 0
|
||||
|
||||
self._tokens.append(t)
|
||||
|
||||
def is_empty(self):
|
||||
return not bool(self._tokens)
|
||||
|
||||
#PLY compliant interface
|
||||
def token(self):
|
||||
if self._current>=len(self._tokens):
|
||||
return None
|
||||
t = self._tokens[self._current]
|
||||
self._current += 1
|
||||
return t
|
||||
|
||||
|
||||
def get_tokens(s):
|
||||
"""Parse the input string and return a tuple (tokens, unprocessed) where
|
||||
tokens is a list of parsed tokens and unprocessed is the part of the input
|
||||
string left untouched by the lexer.
|
||||
"""
|
||||
lexer = PLYLexer()
|
||||
untouched = lexer.add(s, True)
|
||||
tokens = []
|
||||
while 1:
|
||||
token = lexer.token()
|
||||
if token is None:
|
||||
break
|
||||
tokens.append(token)
|
||||
|
||||
tokens = [(t.value, t.type) for t in tokens]
|
||||
return tokens, untouched
|
||||
783
sources/poky/bitbake/lib/bb/pysh/pyshyacc.py
Normal file
783
sources/poky/bitbake/lib/bb/pysh/pyshyacc.py
Normal file
@@ -0,0 +1,783 @@
|
||||
# pyshyacc.py - PLY grammar definition for pysh
|
||||
#
|
||||
# Copyright 2007 Patrick Mezard
|
||||
#
|
||||
# This software may be used and distributed according to the terms
|
||||
# of the GNU General Public License, incorporated herein by reference.
|
||||
|
||||
"""PLY grammar file.
|
||||
"""
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
import bb.pysh.pyshlex as pyshlex
|
||||
tokens = pyshlex.tokens
|
||||
|
||||
from ply import yacc
|
||||
import bb.pysh.sherrors as sherrors
|
||||
|
||||
class IORedirect:
|
||||
def __init__(self, op, filename, io_number=None):
|
||||
self.op = op
|
||||
self.filename = filename
|
||||
self.io_number = io_number
|
||||
|
||||
class HereDocument:
|
||||
def __init__(self, op, name, content, io_number=None):
|
||||
self.op = op
|
||||
self.name = name
|
||||
self.content = content
|
||||
self.io_number = io_number
|
||||
|
||||
def make_io_redirect(p):
|
||||
"""Make an IORedirect instance from the input 'io_redirect' production."""
|
||||
name, io_number, io_target = p
|
||||
assert name=='io_redirect'
|
||||
|
||||
if io_target[0]=='io_file':
|
||||
io_type, io_op, io_file = io_target
|
||||
return IORedirect(io_op, io_file, io_number)
|
||||
elif io_target[0]=='io_here':
|
||||
io_type, io_op, io_name, io_content = io_target
|
||||
return HereDocument(io_op, io_name, io_content, io_number)
|
||||
else:
|
||||
assert False, "Invalid IO redirection token %s" % repr(io_type)
|
||||
|
||||
class SimpleCommand:
|
||||
"""
|
||||
assigns contains (name, value) pairs.
|
||||
"""
|
||||
def __init__(self, words, redirs, assigns):
|
||||
self.words = list(words)
|
||||
self.redirs = list(redirs)
|
||||
self.assigns = list(assigns)
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, commands, reverse_status=False):
|
||||
self.commands = list(commands)
|
||||
assert self.commands #Grammar forbids this
|
||||
self.reverse_status = reverse_status
|
||||
|
||||
class AndOr:
|
||||
def __init__(self, op, left, right):
|
||||
self.op = str(op)
|
||||
self.left = left
|
||||
self.right = right
|
||||
|
||||
class ForLoop:
|
||||
def __init__(self, name, items, cmds):
|
||||
self.name = str(name)
|
||||
self.items = list(items)
|
||||
self.cmds = list(cmds)
|
||||
|
||||
class WhileLoop:
|
||||
def __init__(self, condition, cmds):
|
||||
self.condition = list(condition)
|
||||
self.cmds = list(cmds)
|
||||
|
||||
class UntilLoop:
|
||||
def __init__(self, condition, cmds):
|
||||
self.condition = list(condition)
|
||||
self.cmds = list(cmds)
|
||||
|
||||
class FunDef:
|
||||
def __init__(self, name, body):
|
||||
self.name = str(name)
|
||||
self.body = body
|
||||
|
||||
class BraceGroup:
|
||||
def __init__(self, cmds):
|
||||
self.cmds = list(cmds)
|
||||
|
||||
class IfCond:
|
||||
def __init__(self, cond, if_cmds, else_cmds):
|
||||
self.cond = list(cond)
|
||||
self.if_cmds = if_cmds
|
||||
self.else_cmds = else_cmds
|
||||
|
||||
class Case:
|
||||
def __init__(self, name, items):
|
||||
self.name = name
|
||||
self.items = items
|
||||
|
||||
class SubShell:
|
||||
def __init__(self, cmds):
|
||||
self.cmds = cmds
|
||||
|
||||
class RedirectList:
|
||||
def __init__(self, cmd, redirs):
|
||||
self.cmd = cmd
|
||||
self.redirs = list(redirs)
|
||||
|
||||
def get_production(productions, ptype):
|
||||
"""productions must be a list of production tuples like (name, obj) where
|
||||
name is the production string identifier.
|
||||
Return the first production named 'ptype'. Raise KeyError if None can be
|
||||
found.
|
||||
"""
|
||||
for production in productions:
|
||||
if production is not None and production[0]==ptype:
|
||||
return production
|
||||
raise KeyError(ptype)
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# PLY grammar definition
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
def p_multiple_commands(p):
|
||||
"""multiple_commands : newline_sequence
|
||||
| complete_command
|
||||
| multiple_commands complete_command"""
|
||||
if len(p)==2:
|
||||
if p[1] is not None:
|
||||
p[0] = [p[1]]
|
||||
else:
|
||||
p[0] = []
|
||||
else:
|
||||
p[0] = p[1] + [p[2]]
|
||||
|
||||
def p_complete_command(p):
|
||||
"""complete_command : list separator
|
||||
| list"""
|
||||
if len(p)==3 and p[2] and p[2][1] == '&':
|
||||
p[0] = ('async', p[1])
|
||||
else:
|
||||
p[0] = p[1]
|
||||
|
||||
def p_list(p):
|
||||
"""list : list separator_op and_or
|
||||
| and_or"""
|
||||
if len(p)==2:
|
||||
p[0] = [p[1]]
|
||||
else:
|
||||
#if p[2]!=';':
|
||||
# raise NotImplementedError('AND-OR list asynchronous execution is not implemented')
|
||||
p[0] = p[1] + [p[3]]
|
||||
|
||||
def p_and_or(p):
|
||||
"""and_or : pipeline
|
||||
| and_or AND_IF linebreak pipeline
|
||||
| and_or OR_IF linebreak pipeline"""
|
||||
if len(p)==2:
|
||||
p[0] = p[1]
|
||||
else:
|
||||
p[0] = ('and_or', AndOr(p[2], p[1], p[4]))
|
||||
|
||||
def p_maybe_bang_word(p):
|
||||
"""maybe_bang_word : Bang"""
|
||||
p[0] = ('maybe_bang_word', p[1])
|
||||
|
||||
def p_pipeline(p):
|
||||
"""pipeline : pipe_sequence
|
||||
| bang_word pipe_sequence"""
|
||||
if len(p)==3:
|
||||
p[0] = ('pipeline', Pipeline(p[2][1:], True))
|
||||
else:
|
||||
p[0] = ('pipeline', Pipeline(p[1][1:]))
|
||||
|
||||
def p_pipe_sequence(p):
|
||||
"""pipe_sequence : command
|
||||
| pipe_sequence PIPE linebreak command"""
|
||||
if len(p)==2:
|
||||
p[0] = ['pipe_sequence', p[1]]
|
||||
else:
|
||||
p[0] = p[1] + [p[4]]
|
||||
|
||||
def p_command(p):
|
||||
"""command : simple_command
|
||||
| compound_command
|
||||
| compound_command redirect_list
|
||||
| function_definition"""
|
||||
|
||||
if p[1][0] in ( 'simple_command',
|
||||
'for_clause',
|
||||
'while_clause',
|
||||
'until_clause',
|
||||
'case_clause',
|
||||
'if_clause',
|
||||
'function_definition',
|
||||
'subshell',
|
||||
'brace_group',):
|
||||
if len(p) == 2:
|
||||
p[0] = p[1]
|
||||
else:
|
||||
p[0] = ('redirect_list', RedirectList(p[1], p[2][1:]))
|
||||
else:
|
||||
raise NotImplementedError('%s command is not implemented' % repr(p[1][0]))
|
||||
|
||||
def p_compound_command(p):
|
||||
"""compound_command : brace_group
|
||||
| subshell
|
||||
| for_clause
|
||||
| case_clause
|
||||
| if_clause
|
||||
| while_clause
|
||||
| until_clause"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_subshell(p):
|
||||
"""subshell : LPARENS compound_list RPARENS"""
|
||||
p[0] = ('subshell', SubShell(p[2][1:]))
|
||||
|
||||
def p_compound_list(p):
|
||||
"""compound_list : term
|
||||
| newline_list term
|
||||
| term separator
|
||||
| newline_list term separator"""
|
||||
productions = p[1:]
|
||||
try:
|
||||
sep = get_production(productions, 'separator')
|
||||
if sep[1]!=';':
|
||||
raise NotImplementedError()
|
||||
except KeyError:
|
||||
pass
|
||||
term = get_production(productions, 'term')
|
||||
p[0] = ['compound_list'] + term[1:]
|
||||
|
||||
def p_term(p):
|
||||
"""term : term separator and_or
|
||||
| and_or"""
|
||||
if len(p)==2:
|
||||
p[0] = ['term', p[1]]
|
||||
else:
|
||||
if p[2] is not None and p[2][1] == '&':
|
||||
p[0] = ['term', ('async', p[1][1:])] + [p[3]]
|
||||
else:
|
||||
p[0] = p[1] + [p[3]]
|
||||
|
||||
def p_maybe_for_word(p):
|
||||
# Rearrange 'For' priority wrt TOKEN. See p_for_word
|
||||
"""maybe_for_word : For"""
|
||||
p[0] = ('maybe_for_word', p[1])
|
||||
|
||||
def p_for_clause(p):
|
||||
"""for_clause : for_word name linebreak do_group
|
||||
| for_word name linebreak in sequential_sep do_group
|
||||
| for_word name linebreak in wordlist sequential_sep do_group"""
|
||||
productions = p[1:]
|
||||
do_group = get_production(productions, 'do_group')
|
||||
try:
|
||||
items = get_production(productions, 'in')[1:]
|
||||
except KeyError:
|
||||
raise NotImplementedError('"in" omission is not implemented')
|
||||
|
||||
try:
|
||||
items = get_production(productions, 'wordlist')[1:]
|
||||
except KeyError:
|
||||
items = []
|
||||
|
||||
name = p[2]
|
||||
p[0] = ('for_clause', ForLoop(name, items, do_group[1:]))
|
||||
|
||||
def p_name(p):
|
||||
"""name : token""" #Was NAME instead of token
|
||||
p[0] = p[1]
|
||||
|
||||
def p_in(p):
|
||||
"""in : In"""
|
||||
p[0] = ('in', p[1])
|
||||
|
||||
def p_wordlist(p):
|
||||
"""wordlist : wordlist token
|
||||
| token"""
|
||||
if len(p)==2:
|
||||
p[0] = ['wordlist', ('TOKEN', p[1])]
|
||||
else:
|
||||
p[0] = p[1] + [('TOKEN', p[2])]
|
||||
|
||||
def p_case_clause(p):
|
||||
"""case_clause : Case token linebreak in linebreak case_list Esac
|
||||
| Case token linebreak in linebreak case_list_ns Esac
|
||||
| Case token linebreak in linebreak Esac"""
|
||||
if len(p) < 8:
|
||||
items = []
|
||||
else:
|
||||
items = p[6][1:]
|
||||
name = p[2]
|
||||
p[0] = ('case_clause', Case(name, [c[1] for c in items]))
|
||||
|
||||
def p_case_list_ns(p):
|
||||
"""case_list_ns : case_list case_item_ns
|
||||
| case_item_ns"""
|
||||
p_case_list(p)
|
||||
|
||||
def p_case_list(p):
|
||||
"""case_list : case_list case_item
|
||||
| case_item"""
|
||||
if len(p)==2:
|
||||
p[0] = ['case_list', p[1]]
|
||||
else:
|
||||
p[0] = p[1] + [p[2]]
|
||||
|
||||
def p_case_item_ns(p):
|
||||
"""case_item_ns : pattern RPARENS linebreak
|
||||
| pattern RPARENS compound_list linebreak
|
||||
| LPARENS pattern RPARENS linebreak
|
||||
| LPARENS pattern RPARENS compound_list linebreak"""
|
||||
p_case_item(p)
|
||||
|
||||
def p_case_item(p):
|
||||
"""case_item : pattern RPARENS linebreak DSEMI linebreak
|
||||
| pattern RPARENS compound_list DSEMI linebreak
|
||||
| LPARENS pattern RPARENS linebreak DSEMI linebreak
|
||||
| LPARENS pattern RPARENS compound_list DSEMI linebreak"""
|
||||
if len(p) < 7:
|
||||
name = p[1][1:]
|
||||
else:
|
||||
name = p[2][1:]
|
||||
|
||||
try:
|
||||
cmds = get_production(p[1:], "compound_list")[1:]
|
||||
except KeyError:
|
||||
cmds = []
|
||||
|
||||
p[0] = ('case_item', (name, cmds))
|
||||
|
||||
def p_pattern(p):
|
||||
"""pattern : token
|
||||
| pattern PIPE token"""
|
||||
if len(p)==2:
|
||||
p[0] = ['pattern', ('TOKEN', p[1])]
|
||||
else:
|
||||
p[0] = p[1] + [('TOKEN', p[2])]
|
||||
|
||||
def p_maybe_if_word(p):
|
||||
# Rearrange 'If' priority wrt TOKEN. See p_if_word
|
||||
"""maybe_if_word : If"""
|
||||
p[0] = ('maybe_if_word', p[1])
|
||||
|
||||
def p_maybe_then_word(p):
|
||||
# Rearrange 'Then' priority wrt TOKEN. See p_then_word
|
||||
"""maybe_then_word : Then"""
|
||||
p[0] = ('maybe_then_word', p[1])
|
||||
|
||||
def p_if_clause(p):
|
||||
"""if_clause : if_word compound_list then_word compound_list else_part Fi
|
||||
| if_word compound_list then_word compound_list Fi"""
|
||||
else_part = []
|
||||
if len(p)==7:
|
||||
else_part = p[5]
|
||||
p[0] = ('if_clause', IfCond(p[2][1:], p[4][1:], else_part))
|
||||
|
||||
def p_else_part(p):
|
||||
"""else_part : Elif compound_list then_word compound_list else_part
|
||||
| Elif compound_list then_word compound_list
|
||||
| Else compound_list"""
|
||||
if len(p)==3:
|
||||
p[0] = p[2][1:]
|
||||
else:
|
||||
else_part = []
|
||||
if len(p)==6:
|
||||
else_part = p[5]
|
||||
p[0] = ('elif', IfCond(p[2][1:], p[4][1:], else_part))
|
||||
|
||||
def p_while_clause(p):
|
||||
"""while_clause : While compound_list do_group"""
|
||||
p[0] = ('while_clause', WhileLoop(p[2][1:], p[3][1:]))
|
||||
|
||||
def p_maybe_until_word(p):
|
||||
# Rearrange 'Until' priority wrt TOKEN. See p_until_word
|
||||
"""maybe_until_word : Until"""
|
||||
p[0] = ('maybe_until_word', p[1])
|
||||
|
||||
def p_until_clause(p):
|
||||
"""until_clause : until_word compound_list do_group"""
|
||||
p[0] = ('until_clause', UntilLoop(p[2][1:], p[3][1:]))
|
||||
|
||||
def p_function_definition(p):
|
||||
"""function_definition : fname LPARENS RPARENS linebreak function_body"""
|
||||
p[0] = ('function_definition', FunDef(p[1], p[5]))
|
||||
|
||||
def p_function_body(p):
|
||||
"""function_body : compound_command
|
||||
| compound_command redirect_list"""
|
||||
if len(p)!=2:
|
||||
raise NotImplementedError('functions redirections lists are not implemented')
|
||||
p[0] = p[1]
|
||||
|
||||
def p_fname(p):
|
||||
"""fname : TOKEN""" #Was NAME instead of token
|
||||
p[0] = p[1]
|
||||
|
||||
def p_brace_group(p):
|
||||
"""brace_group : Lbrace compound_list Rbrace"""
|
||||
p[0] = ('brace_group', BraceGroup(p[2][1:]))
|
||||
|
||||
def p_maybe_done_word(p):
|
||||
#See p_assignment_word for details.
|
||||
"""maybe_done_word : Done"""
|
||||
p[0] = ('maybe_done_word', p[1])
|
||||
|
||||
def p_maybe_do_word(p):
|
||||
"""maybe_do_word : Do"""
|
||||
p[0] = ('maybe_do_word', p[1])
|
||||
|
||||
def p_do_group(p):
|
||||
"""do_group : do_word compound_list done_word"""
|
||||
#Do group contains a list of AndOr
|
||||
p[0] = ['do_group'] + p[2][1:]
|
||||
|
||||
def p_simple_command(p):
|
||||
"""simple_command : cmd_prefix cmd_word cmd_suffix
|
||||
| cmd_prefix cmd_word
|
||||
| cmd_prefix
|
||||
| cmd_name cmd_suffix
|
||||
| cmd_name"""
|
||||
words, redirs, assigns = [], [], []
|
||||
for e in p[1:]:
|
||||
name = e[0]
|
||||
if name in ('cmd_prefix', 'cmd_suffix'):
|
||||
for sube in e[1:]:
|
||||
subname = sube[0]
|
||||
if subname=='io_redirect':
|
||||
redirs.append(make_io_redirect(sube))
|
||||
elif subname=='ASSIGNMENT_WORD':
|
||||
assigns.append(sube)
|
||||
else:
|
||||
words.append(sube)
|
||||
elif name in ('cmd_word', 'cmd_name'):
|
||||
words.append(e)
|
||||
|
||||
cmd = SimpleCommand(words, redirs, assigns)
|
||||
p[0] = ('simple_command', cmd)
|
||||
|
||||
def p_cmd_name(p):
|
||||
"""cmd_name : TOKEN"""
|
||||
p[0] = ('cmd_name', p[1])
|
||||
|
||||
def p_cmd_word(p):
|
||||
"""cmd_word : token"""
|
||||
p[0] = ('cmd_word', p[1])
|
||||
|
||||
def p_maybe_assignment_word(p):
|
||||
#See p_assignment_word for details.
|
||||
"""maybe_assignment_word : ASSIGNMENT_WORD"""
|
||||
p[0] = ('maybe_assignment_word', p[1])
|
||||
|
||||
def p_cmd_prefix(p):
|
||||
"""cmd_prefix : io_redirect
|
||||
| cmd_prefix io_redirect
|
||||
| assignment_word
|
||||
| cmd_prefix assignment_word"""
|
||||
try:
|
||||
prefix = get_production(p[1:], 'cmd_prefix')
|
||||
except KeyError:
|
||||
prefix = ['cmd_prefix']
|
||||
|
||||
try:
|
||||
value = get_production(p[1:], 'assignment_word')[1]
|
||||
value = ('ASSIGNMENT_WORD', value.split('=', 1))
|
||||
except KeyError:
|
||||
value = get_production(p[1:], 'io_redirect')
|
||||
p[0] = prefix + [value]
|
||||
|
||||
def p_cmd_suffix(p):
|
||||
"""cmd_suffix : io_redirect
|
||||
| cmd_suffix io_redirect
|
||||
| token
|
||||
| cmd_suffix token
|
||||
| maybe_for_word
|
||||
| cmd_suffix maybe_for_word
|
||||
| maybe_done_word
|
||||
| cmd_suffix maybe_done_word
|
||||
| maybe_do_word
|
||||
| cmd_suffix maybe_do_word
|
||||
| maybe_until_word
|
||||
| cmd_suffix maybe_until_word
|
||||
| maybe_assignment_word
|
||||
| cmd_suffix maybe_assignment_word
|
||||
| maybe_if_word
|
||||
| cmd_suffix maybe_if_word
|
||||
| maybe_then_word
|
||||
| cmd_suffix maybe_then_word
|
||||
| maybe_bang_word
|
||||
| cmd_suffix maybe_bang_word"""
|
||||
try:
|
||||
suffix = get_production(p[1:], 'cmd_suffix')
|
||||
token = p[2]
|
||||
except KeyError:
|
||||
suffix = ['cmd_suffix']
|
||||
token = p[1]
|
||||
|
||||
if isinstance(token, tuple):
|
||||
if token[0]=='io_redirect':
|
||||
p[0] = suffix + [token]
|
||||
else:
|
||||
#Convert maybe_* to TOKEN if necessary
|
||||
p[0] = suffix + [('TOKEN', token[1])]
|
||||
else:
|
||||
p[0] = suffix + [('TOKEN', token)]
|
||||
|
||||
def p_redirect_list(p):
|
||||
"""redirect_list : io_redirect
|
||||
| redirect_list io_redirect"""
|
||||
if len(p) == 2:
|
||||
p[0] = ['redirect_list', make_io_redirect(p[1])]
|
||||
else:
|
||||
p[0] = p[1] + [make_io_redirect(p[2])]
|
||||
|
||||
def p_io_redirect(p):
|
||||
"""io_redirect : io_file
|
||||
| IO_NUMBER io_file
|
||||
| io_here
|
||||
| IO_NUMBER io_here"""
|
||||
if len(p)==3:
|
||||
p[0] = ('io_redirect', p[1], p[2])
|
||||
else:
|
||||
p[0] = ('io_redirect', None, p[1])
|
||||
|
||||
def p_io_file(p):
|
||||
#Return the tuple (operator, filename)
|
||||
"""io_file : LESS filename
|
||||
| LESSAND filename
|
||||
| GREATER filename
|
||||
| GREATAND filename
|
||||
| DGREAT filename
|
||||
| LESSGREAT filename
|
||||
| CLOBBER filename"""
|
||||
#Extract the filename from the file
|
||||
p[0] = ('io_file', p[1], p[2][1])
|
||||
|
||||
def p_filename(p):
|
||||
#Return the filename
|
||||
"""filename : TOKEN"""
|
||||
p[0] = ('filename', p[1])
|
||||
|
||||
def p_io_here(p):
|
||||
"""io_here : DLESS here_end
|
||||
| DLESSDASH here_end"""
|
||||
p[0] = ('io_here', p[1], p[2][1], p[2][2])
|
||||
|
||||
def p_here_end(p):
|
||||
"""here_end : HERENAME TOKEN"""
|
||||
p[0] = ('here_document', p[1], p[2])
|
||||
|
||||
def p_newline_sequence(p):
|
||||
# Nothing in the grammar can handle leading NEWLINE productions, so add
|
||||
# this one with the lowest possible priority relatively to newline_list.
|
||||
"""newline_sequence : newline_list"""
|
||||
p[0] = None
|
||||
|
||||
def p_newline_list(p):
|
||||
"""newline_list : NEWLINE
|
||||
| newline_list NEWLINE"""
|
||||
p[0] = None
|
||||
|
||||
def p_linebreak(p):
|
||||
"""linebreak : newline_list
|
||||
| empty"""
|
||||
p[0] = None
|
||||
|
||||
def p_separator_op(p):
|
||||
"""separator_op : COMMA
|
||||
| COMMA COMMA
|
||||
| AMP"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_separator(p):
|
||||
"""separator : separator_op linebreak
|
||||
| newline_list"""
|
||||
if len(p)==2:
|
||||
#Ignore newlines
|
||||
p[0] = None
|
||||
else:
|
||||
#Keep the separator operator
|
||||
p[0] = ('separator', p[1])
|
||||
|
||||
def p_sequential_sep(p):
|
||||
"""sequential_sep : COMMA linebreak
|
||||
| newline_list"""
|
||||
p[0] = None
|
||||
|
||||
# Low priority TOKEN => for_word conversion.
|
||||
# Let maybe_for_word be used as a token when necessary in higher priority
|
||||
# rules.
|
||||
def p_for_word(p):
|
||||
"""for_word : maybe_for_word"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_if_word(p):
|
||||
"""if_word : maybe_if_word"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_then_word(p):
|
||||
"""then_word : maybe_then_word"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_done_word(p):
|
||||
"""done_word : maybe_done_word"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_do_word(p):
|
||||
"""do_word : maybe_do_word"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_until_word(p):
|
||||
"""until_word : maybe_until_word"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_assignment_word(p):
|
||||
"""assignment_word : maybe_assignment_word"""
|
||||
p[0] = ('assignment_word', p[1][1])
|
||||
|
||||
def p_bang_word(p):
|
||||
"""bang_word : maybe_bang_word"""
|
||||
p[0] = ('bang_word', p[1][1])
|
||||
|
||||
def p_token(p):
|
||||
"""token : TOKEN
|
||||
| Fi"""
|
||||
p[0] = p[1]
|
||||
|
||||
def p_empty(p):
|
||||
'empty :'
|
||||
p[0] = None
|
||||
|
||||
# Error rule for syntax errors
|
||||
def p_error(p):
|
||||
msg = []
|
||||
w = msg.append
|
||||
if p:
|
||||
w('%r\n' % p)
|
||||
w('followed by:\n')
|
||||
for i in range(5):
|
||||
n = yacc.token()
|
||||
if not n:
|
||||
break
|
||||
w(' %r\n' % n)
|
||||
else:
|
||||
w('Unexpected EOF')
|
||||
raise sherrors.ShellSyntaxError(''.join(msg))
|
||||
|
||||
# Build the parser
|
||||
try:
|
||||
import pyshtables
|
||||
except ImportError:
|
||||
outputdir = os.path.dirname(__file__)
|
||||
if not os.access(outputdir, os.W_OK):
|
||||
outputdir = ''
|
||||
yacc.yacc(tabmodule = 'pyshtables', outputdir = outputdir, debug = 0)
|
||||
else:
|
||||
yacc.yacc(tabmodule = 'pysh.pyshtables', write_tables = 0, debug = 0)
|
||||
|
||||
|
||||
def parse(input, eof=False, debug=False):
|
||||
"""Parse a whole script at once and return the generated AST and unconsumed
|
||||
data in a tuple.
|
||||
|
||||
NOTE: eof is probably meaningless for now, the parser being unable to work
|
||||
in pull mode. It should be set to True.
|
||||
"""
|
||||
lexer = pyshlex.PLYLexer()
|
||||
remaining = lexer.add(input, eof)
|
||||
if lexer.is_empty():
|
||||
return [], remaining
|
||||
if debug:
|
||||
debug = 2
|
||||
return yacc.parse(lexer=lexer, debug=debug), remaining
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# AST rendering helpers
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
def format_commands(v):
|
||||
"""Return a tree made of strings and lists. Make command trees easier to
|
||||
display.
|
||||
"""
|
||||
if isinstance(v, list):
|
||||
return [format_commands(c) for c in v]
|
||||
if isinstance(v, tuple):
|
||||
if len(v)==2 and isinstance(v[0], str) and not isinstance(v[1], str):
|
||||
if v[0] == 'async':
|
||||
return ['AsyncList', map(format_commands, v[1])]
|
||||
else:
|
||||
#Avoid decomposing tuples like ('pipeline', Pipeline(...))
|
||||
return format_commands(v[1])
|
||||
return format_commands(list(v))
|
||||
elif isinstance(v, IfCond):
|
||||
name = ['IfCond']
|
||||
name += ['if', map(format_commands, v.cond)]
|
||||
name += ['then', map(format_commands, v.if_cmds)]
|
||||
name += ['else', map(format_commands, v.else_cmds)]
|
||||
return name
|
||||
elif isinstance(v, ForLoop):
|
||||
name = ['ForLoop']
|
||||
name += [repr(v.name)+' in ', map(str, v.items)]
|
||||
name += ['commands', map(format_commands, v.cmds)]
|
||||
return name
|
||||
elif isinstance(v, AndOr):
|
||||
return [v.op, format_commands(v.left), format_commands(v.right)]
|
||||
elif isinstance(v, Pipeline):
|
||||
name = 'Pipeline'
|
||||
if v.reverse_status:
|
||||
name = '!' + name
|
||||
return [name, format_commands(v.commands)]
|
||||
elif isinstance(v, Case):
|
||||
name = ['Case']
|
||||
name += [v.name, format_commands(v.items)]
|
||||
elif isinstance(v, SimpleCommand):
|
||||
name = ['SimpleCommand']
|
||||
if v.words:
|
||||
name += ['words', map(str, v.words)]
|
||||
if v.assigns:
|
||||
assigns = [tuple(a[1]) for a in v.assigns]
|
||||
name += ['assigns', map(str, assigns)]
|
||||
if v.redirs:
|
||||
name += ['redirs', map(format_commands, v.redirs)]
|
||||
return name
|
||||
elif isinstance(v, RedirectList):
|
||||
name = ['RedirectList']
|
||||
if v.redirs:
|
||||
name += ['redirs', map(format_commands, v.redirs)]
|
||||
name += ['command', format_commands(v.cmd)]
|
||||
return name
|
||||
elif isinstance(v, IORedirect):
|
||||
return ' '.join(map(str, (v.io_number, v.op, v.filename)))
|
||||
elif isinstance(v, HereDocument):
|
||||
return ' '.join(map(str, (v.io_number, v.op, repr(v.name), repr(v.content))))
|
||||
elif isinstance(v, SubShell):
|
||||
return ['SubShell', map(format_commands, v.cmds)]
|
||||
else:
|
||||
return repr(v)
|
||||
|
||||
def print_commands(cmds, output=sys.stdout):
|
||||
"""Pretty print a command tree."""
|
||||
def print_tree(cmd, spaces, output):
|
||||
if isinstance(cmd, list):
|
||||
for c in cmd:
|
||||
print_tree(c, spaces + 3, output)
|
||||
else:
|
||||
print >>output, ' '*spaces + str(cmd)
|
||||
|
||||
formatted = format_commands(cmds)
|
||||
print_tree(formatted, 0, output)
|
||||
|
||||
|
||||
def stringify_commands(cmds):
|
||||
"""Serialize a command tree as a string.
|
||||
|
||||
Returned string is not pretty and is currently used for unit tests only.
|
||||
"""
|
||||
def stringify(value):
|
||||
output = []
|
||||
if isinstance(value, list):
|
||||
formatted = []
|
||||
for v in value:
|
||||
formatted.append(stringify(v))
|
||||
formatted = ' '.join(formatted)
|
||||
output.append(''.join(['<', formatted, '>']))
|
||||
else:
|
||||
output.append(value)
|
||||
return ' '.join(output)
|
||||
|
||||
return stringify(format_commands(cmds))
|
||||
|
||||
|
||||
def visit_commands(cmds, callable):
|
||||
"""Visit the command tree and execute callable on every Pipeline and
|
||||
SimpleCommand instances.
|
||||
"""
|
||||
if isinstance(cmds, (tuple, list)):
|
||||
map(lambda c: visit_commands(c,callable), cmds)
|
||||
elif isinstance(cmds, (Pipeline, SimpleCommand)):
|
||||
callable(cmds)
|
||||
15
sources/poky/bitbake/lib/bb/pysh/sherrors.py
Normal file
15
sources/poky/bitbake/lib/bb/pysh/sherrors.py
Normal file
@@ -0,0 +1,15 @@
|
||||
# sherrors.py - shell errors and signals
|
||||
#
|
||||
# Copyright 2007 Patrick Mezard
|
||||
#
|
||||
# This software may be used and distributed according to the terms
|
||||
# of the GNU General Public License, incorporated herein by reference.
|
||||
|
||||
"""Define shell exceptions and error codes.
|
||||
"""
|
||||
|
||||
class ShellError(Exception):
|
||||
pass
|
||||
|
||||
class ShellSyntaxError(ShellError):
|
||||
pass
|
||||
Reference in New Issue
Block a user