Commit 13196704 authored by Tom Niget's avatar Tom Niget

Initial transpiler PoC. Supports most usual Python structures, and a basic...

Initial transpiler PoC. Supports most usual Python structures, and a basic builtins module is provided.
parent 6a97f794
---
Language: Cpp
BasedOnStyle: LLVM
IndentWidth: 2
UseTab: Never
---
\ No newline at end of file
Checks: '-google-explicit-constructor -misc-no-recursion'
\ No newline at end of file
.idea
\ No newline at end of file
*
!/**/
!*.*
!Makefile
*.o
\ No newline at end of file
CXX=g++-11 CXX=g++-11
CXXFLAGS=-g -O3 -fwhole-program -flto -Wall -Wextra -Werror -std=c++20 -fcoroutines -I../include CXXFLAGS=-g -O3 -fwhole-program -flto -Wall -Wextra -Werror -std=c++20 -fcoroutines -I../include
LDLIBS=-pthread -luring -lfmt LDLIBS=-pthread -luring -lfmt
%.o: %.cpp
$(CXX) -c $(CXXFLAGS) $< -o $@
%: %.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
cat: cat.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
fork: fork.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
fork_exceptions: fork_exceptions.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
fork_variations: fork_variations.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
future_awaited: future_awaited.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
future_detached: future_detached.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
loop: loop.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
naive_fibonacci_fork: naive_fibonacci_fork.cpp
$(CXX) $(CXXFLAGS) $< -o $@ $(LDLIBS)
\ No newline at end of file
//
// Created by Tom on 07/03/2023.
//
#ifndef TYPON_BUILTINS_HPP
#define TYPON_BUILTINS_HPP
#include <string>
#include <vector>
#include <iostream>
#include <ostream>
#include <unordered_set>
#include <optional>
#include <sstream>
#include <unordered_map>
using namespace std::literals;
template<typename T>
concept PyIterator = requires(T t) {
{ t.py_next() } -> std::same_as<std::optional<T>>;
};
template<typename T>
concept PyIterable = requires(T t) {
{ t.py_iter() } -> PyIterator;
};
template<PyIterable T, PyIterator U>
U iter(const T &t) {
return t.py_iter();
}
template<typename T>
concept PyLen = requires(const T &t) {
{ t.py_len() } -> std::same_as<size_t>;
};
template<PyLen T>
size_t len(const T &t) {
return t.py_len();
}
template<typename T>
void print_to(const T &x, std::ostream &s) {
s << x;
}
template<typename T>
concept Boolean = std::same_as<T, bool>;
template<typename T>
concept Printable = requires(const T &x, std::ostream &s) {
{ x.py_print(s) } -> std::same_as<void>;
};
template<Printable T>
void print_to(const T &x, std::ostream &s) {
x.py_print(s);
}
template<Boolean T>
void print_to(const T &x, std::ostream &s) {
s << (x ? "True" : "False");
}
template<typename T>
std::string str(const T &x) {
std::stringstream s;
print_to(x, s);
return s.str();
}
template<typename T>
class PyList : public std::vector<T> {
public:
PyList(std::vector<T> &&v) : std::vector<T>(std::move(v)) {}
PyList(std::initializer_list<T> &&v) : std::vector<T>(std::move(v)) {}
operator std::vector<T>() const {
return std::vector<T>(this->begin(), this->end());
}
operator std::vector<T> &() {
return *reinterpret_cast<std::vector<T> *>(this);
}
size_t py_len() const {
return this->size();
}
void py_print(std::ostream &s) const {
s << '[';
if (this->size() > 0) {
print_to(this->operator[](0), s);
for (size_t i = 1; i < this->size(); i++) {
s << ", ";
print_to(this->operator[](i), s);
}
}
s << ']';
}
PyList<T> operator+(const PyList<T> &other) const {
std::vector<T> v;
v.reserve(this->size() + other.size());
v.insert(v.end(), this->begin(), this->end());
v.insert(v.end(), other.begin(), other.end());
return PyList<T>(std::move(v));
}
PyList<T> operator*(size_t n) const {
std::vector<T> v;
v.reserve(this->size() * n);
for (size_t i = 0; i < n; i++) {
v.insert(v.end(), this->begin(), this->end());
}
return PyList<T>(std::move(v));
}
};
template<typename T>
class PySet : public std::unordered_set<T> {
public:
PySet(std::unordered_set<T> &&s) : std::unordered_set<T>(std::move(s)) {}
PySet(std::initializer_list<T> &&s) : std::unordered_set<T>(std::move(s)) {}
operator std::unordered_set<T>() const {
return std::unordered_set<T>(this->begin(), this->end());
}
size_t py_len() const {
return this->size();
}
bool py_contains(const T &t) const {
return this->find(t) != this->end();
}
class iterator {
public:
using value_type = T;
using difference_type = std::ptrdiff_t;
using pointer = T *;
using reference = T &;
using iterator_category = std::forward_iterator_tag;
iterator(typename std::unordered_set<T>::iterator it) : _it(it) {}
iterator &operator++() {
_it++;
return *this;
}
iterator operator++(int) {
iterator tmp = *this;
_it++;
return tmp;
}
bool operator==(const iterator &rhs) const {
return _it == rhs._it;
}
bool operator!=(const iterator &rhs) const {
return _it != rhs._it;
}
const T &operator*() const {
return *_it;
}
const T *operator->() const {
return &*_it;
}
private:
typename std::unordered_set<T>::iterator _it;
};
iterator py_iter() const {
return this->begin();
}
void py_print(std::ostream &s) const {
s << '{';
if (this->size() > 0) {
print_to(*this->begin(), s);
for (auto it = ++this->begin(); it != this->end(); it++) {
s << ", ";
print_to(*it, s);
}
}
s << '}';
}
};
template<typename K, typename V>
class PyDict : public std::unordered_map<K, V> {
public:
PyDict(std::unordered_map<K, V> &&m) : std::unordered_map<K, V>(std::move(m)) {}
PyDict(std::initializer_list<std::pair<const K, V>> m) : std::unordered_map<K, V>(m) {}
operator std::unordered_map<K, V>() const {
return std::unordered_map<K, V>(this->begin(), this->end());
}
operator std::unordered_map<K, V> &() {
return *reinterpret_cast<std::unordered_map<K, V> *>(this);
}
size_t py_len() const {
return this->size();
}
bool py_contains(const K &k) const {
return this->find(k) != this->end();
}
class iterator {
public:
using value_type = std::pair<K, V>;
using difference_type = std::ptrdiff_t;
using pointer = std::pair<K, V> *;
using reference = std::pair<K, V> &;
using iterator_category = std::forward_iterator_tag;
iterator(typename std::unordered_map<K, V>::iterator it) : _it(it) {}
iterator &operator++() {
_it++;
return *this;
}
iterator operator++(int) {
iterator tmp = *this;
_it++;
return tmp;
}
bool operator==(const iterator &rhs) const {
return _it == rhs._it;
}
bool operator!=(const iterator &rhs) const {
return _it != rhs._it;
}
const std::pair<K, V> &operator*() const {
return *_it;
}
const std::pair<K, V> *operator->() const {
return &*_it;
}
private:
typename std::unordered_map<K, V>::iterator _it;
};
iterator py_iter() const {
return this->begin();
}
void py_print(std::ostream &s) const {
s << '{';
if (this->size() > 0) {
print_to(this->begin()->first, s);
s << ": ";
print_to(this->begin()->second, s);
for (auto it = ++this->begin(); it != this->end(); it++) {
s << ", ";
print_to(it->first, s);
s << ": ";
print_to(it->second, s);
}
}
s << '}';
}
};
template<typename K, typename V>
PyDict(std::initializer_list<std::pair<K, V>>) -> PyDict<K, V>;
void print() {
std::cout << '\n';
}
template<typename T, typename ... Args>
void print(T const &head, Args const &... args) {
print_to(head, std::cout);
(((std::cout << ' '), print_to(args, std::cout)), ...);
std::cout << '\n';
}
bool is_cpp() {
return true;
}
#endif //TYPON_BUILTINS_HPP
__pycache__
\ No newline at end of file
print("Main")
\ No newline at end of file
Bclang-format==15.0.7 Bclang-format==15.0.7
# coding: utf-8
from os import system
from pathlib import Path
from transpiler import transpile
from transpiler.format import format_code
# print(format_code("int x = 2 + ((3 * 5));;;"))
def run_tests():
for path in Path('tests').glob('*.py'):
print(path.name)
with open(path, "r", encoding="utf-8") as f:
res = format_code(transpile(f.read()))
name_cpp = path.with_suffix('.cpp')
with open(name_cpp, "w", encoding="utf-8") as fcpp:
fcpp.write(res)
name_bin = path.with_suffix('').as_posix()
cmd = f"bash -c 'g++ -I../rt/include -std=c++20 -o {name_bin} {name_cpp.as_posix()}'"
commands = [
cmd,
f"bash -c 'PYTHONPATH=. python3 ./{path.as_posix()}'",
f"bash -c './{name_bin}'",
f"scp ./{name_bin} tom@192.168.139.128:/tmp/{name_bin}", # TODO: temporary test suite. Will fix.
f"bash -c 'ssh tom@192.168.139.128 \"chmod +x /tmp/{name_bin} && /tmp/{name_bin}\"'"
]
for cmd in commands:
if system(cmd) != 0:
print(f"Error running command: {cmd}")
break
if __name__ == "__main__":
run_tests()
*
!/**/
!*.*
*.cpp
\ No newline at end of file
# coding: utf-8
from typon import is_cpp
if __name__ == "__main__":
# todo: 0x55 & 7 == 5
print("C++ " if is_cpp() else "Python",
"res=", 5, ".", True, [4, 5, 6], {7, 8, 9}, [1, 2] + [3, 4], [5, 6] * 3, {1: 7, 9: 3}, 0x55 & 7 == 5)
print()
\ No newline at end of file
from typon import fork, sync
#def fibo(n: int) -> int:
# if n < 2:
# return n
# a = fork(lambda: fibo(n - 1))
# b = fork(lambda: fibo(n - 2))
# sync()
# return a + b
if __name__ == "__main__":
print("res=", 5, ".")
\ No newline at end of file
# coding: utf-8
import ast
from itertools import chain, zip_longest
from typing import *
def compare_ast(node1: Union[ast.expr, list[ast.expr]], node2: Union[ast.expr, list[ast.expr]]) -> bool:
if type(node1) is not type(node2):
return False
if isinstance(node1, ast.AST):
for k, v in vars(node1).items():
if k in {"lineno", "end_lineno", "col_offset", "end_col_offset", "ctx"}:
continue
if not compare_ast(v, getattr(node2, k)):
return False
return True
elif isinstance(node1, list) and isinstance(node2, list):
return all(compare_ast(n1, n2) for n1, n2 in zip_longest(node1, node2))
else:
return node1 == node2
def flatmap(f, items):
return chain.from_iterable(map(f, items))
def join(sep: str, items: Iterable[Iterable[str]]) -> Iterable[str]:
items = iter(items)
try:
yield from next(items)
for item in items:
yield sep
yield from item
except StopIteration:
return
def transpile(source):
tree = ast.parse(source)
# print(ast.unparse(tree))
return "\n".join(filter(None, map(str, TyponVisitor().visit(tree))))
SYMBOLS = {
ast.Eq: "==",
ast.NotEq: '!=',
ast.Pass: '/* pass */',
ast.Mult: '*',
ast.Add: '+',
ast.Sub: '-',
ast.Div: '/',
ast.FloorDiv: '/',
ast.Mod: '%',
ast.Lt: '<',
ast.Gt: '>',
ast.GtE: '>=',
ast.LtE: '<=',
ast.LShift: '<<',
ast.RShift: '>>',
ast.BitXor: '^',
ast.BitOr: '|',
ast.BitAnd: '&',
ast.Not: '!',
ast.IsNot: '!=',
ast.USub: '-',
ast.And: '&&',
ast.Or: '||'
}
"""Mapping of Python AST nodes to C++ symbols."""
PRECEDENCE = [
("*", "/", "%",),
("+", "-"),
("<<", ">>"),
("<", "<=", ">", ">="),
("==", "!="),
("&",),
("^",),
("|",),
("&&",),
("||",),
]
"""Precedence of C++ operators."""
PRECEDENCE_LEVELS = {op: i for i, ops in enumerate(PRECEDENCE) for op in ops}
"""Mapping of C++ operators to their precedence level."""
MAPPINGS = {
"True": "true",
"False": "false",
"None": "nullptr"
}
"""Mapping of Python builtin constants to C++ equivalents."""
class NodeVisitor:
def visit(self, node):
"""Visit a node."""
if type(node) in SYMBOLS:
yield SYMBOLS[type(node)]
else:
for parent in node.__class__.__mro__:
if visitor := getattr(self, 'visit_' + parent.__name__, None):
yield from visitor(node)
break
else:
raise NotImplementedError(node.__class__.__mro__, ast.dump(node))
def process_args(self, node: ast.arguments) -> (str, str):
for field in ("posonlyargs", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults"):
if getattr(node, field, None):
raise NotImplementedError(node, field)
if not node.args:
return "", "()"
f_args = [(arg, f"T{i + 1}") for i, arg in enumerate(node.args)]
return (
"<" + ", ".join(f"typename {t}" for _, t in f_args) + ">",
"(" + ", ".join(f"{t} {next(self.visit(n))}" for n, t in f_args) + ")"
)
def visit_arg(self, node: ast.arg) -> Iterable[str]:
# TODO: identifiers
yield node.arg
class ExpressionVisitor(NodeVisitor):
def __init__(self, precedence: int = 0):
self._precedence = precedence
def visit_Tuple(self, node: ast.Tuple) -> Iterable[str]:
yield f"std::make_tuple({', '.join(flatmap(self.visit, node.elts))})"
def visit_Constant(self, node: ast.Constant) -> Iterable[str]:
if isinstance(node.value, str):
# TODO: escape sequences
yield f"\"{repr(node.value)[1:-1]}\"s"
elif isinstance(node.value, bool):
yield str(node.value).lower()
elif isinstance(node.value, int):
# TODO: bigints
yield str(node.value)
else:
raise NotImplementedError(node, type(node))
def visit_Name(self, node: ast.Name) -> Iterable[str]:
if node.id.startswith("__") and node.id.endswith("__"):
return f"py_{node.id[2:-2]}"
yield MAPPINGS.get(node.id, node.id)
def visit_Compare(self, node: ast.Compare) -> Iterable[str]:
# TODO: operator precedence
operands = [node.left, *node.comparators]
yield from self.visit_binary_operation(node.ops[0], operands[0], operands[1])
for (left, right), op in zip(zip(operands[1:], operands[2:]), node.ops[1:]):
# TODO: cleaner code
yield " && "
yield from self.visit_binary_operation(op, left, right)
def visit_Call(self, node: ast.Call) -> Iterable[str]:
if getattr(node, "keywords", None):
raise NotImplementedError(node, "keywords")
if getattr(node, "starargs", None):
raise NotImplementedError(node, "varargs")
if getattr(node, "kwargs", None):
raise NotImplementedError(node, "kwargs")
yield from self.visit(node.func)
yield "("
yield from join(", ", map(self.visit, node.args))
yield ")"
def visit_Lambda(self, node: ast.Lambda) -> Iterable[str]:
yield "[]"
templ, args = self.process_args(node.args)
yield templ
yield args
yield "{"
yield "return"
yield from self.visit(node.body)
yield ";"
yield "}"
def visit_BinOp(self, node: ast.BinOp) -> Iterable[str]:
yield from self.visit_binary_operation(node.op, node.left, node.right)
def visit_binary_operation(self, op, left: ast.AST, right: ast.AST) -> Iterable[str]:
# TODO: precedence
op = SYMBOLS[type(op)]
inner = ExpressionVisitor(PRECEDENCE_LEVELS[op])
prio = inner._precedence > self._precedence != 0
if prio:
yield "("
yield from inner.visit(left)
yield op
yield from inner.visit(right)
if prio:
yield ")"
def visit_Attribute(self, node: ast.Attribute) -> Iterable[str]:
yield from self.visit(node.value)
yield "."
yield node.attr
def visit_List(self, node: ast.List) -> Iterable[str]:
yield "PyList{"
yield from join(", ", map(self.visit, node.elts))
yield "}"
def visit_Set(self, node: ast.Set) -> Iterable[str]:
yield "PySet{"
yield from join(", ", map(self.visit, node.elts))
yield "}"
def visit_Dict(self, node: ast.Dict) -> Iterable[str]:
def visit_item(key, value):
yield "std::pair {"
yield from self.visit(key)
yield ", "
yield from self.visit(value)
yield "}"
yield "PyDict{"
yield from join(", ", map(visit_item, node.keys, node.values))
yield "}"
def visit_Subscript(self, node: ast.Subscript) -> Iterable[str]:
yield from self.visit(node.value)
yield "["
yield from self.visit(node.slice)
yield "]"
def visit_UnaryOp(self, node: ast.UnaryOp) -> Iterable[str]:
yield from self.visit(node.op)
yield from self.visit(node.operand)
def visit_IfExp(self, node: ast.IfExp) -> Iterable[str]:
yield from self.visit(node.test)
yield " ? "
yield from self.visit(node.body)
yield " : "
yield from self.visit(node.orelse)
class TyponVisitor(NodeVisitor):
def visit_Module(self, node: ast.Module) -> Iterable[str]:
stmt: ast.AST
yield "#include <python/builtins.hpp>"
for stmt in node.body:
yield from self.visit(stmt)
def visit_Expr(self, node: ast.Expr) -> Iterable[str]:
yield from ExpressionVisitor().visit(node.value)
yield ";"
def visit_Import(self, node: ast.Import) -> Iterable[str]:
for name in node.names:
if name == "typon":
yield "// typon import"
else:
raise NotImplementedError(node)
def visit_ImportFrom(self, node: ast.ImportFrom) -> Iterable[str]:
if node.module == "typon":
yield "// typon import"
else:
raise NotImplementedError(node)
def visit_FunctionDef(self, node: ast.FunctionDef) -> Iterable[str]:
templ, args = self.process_args(node.args)
if templ:
yield "template"
yield templ
yield f"auto {node.name}"
yield args
yield "{"
for child in node.body:
yield from self.visit(child)
yield "}"
def visit_If(self, node: ast.If) -> Iterable[str]:
if not node.orelse and compare_ast(node.test, ast.parse('__name__ == "__main__"', mode="eval").body):
yield "int main() {"
for child in node.body:
yield from self.visit(child)
yield "}"
return
yield "if ("
yield from ExpressionVisitor().visit(node.test)
yield ") {"
for child in node.body:
yield from self.visit(child)
yield "}"
if node.orelse:
yield "else "
if isinstance(node.orelse, ast.If):
yield from self.visit(node.orelse)
else:
yield "{"
for child in node.orelse:
yield from self.visit(child)
yield "}"
def visit_Return(self, node: ast.Return) -> Iterable[str]:
yield "return "
if node.value:
yield from ExpressionVisitor().visit(node.value)
yield ";"
def visit_While(self, node: ast.While) -> Iterable[str]:
yield "while ("
yield from ExpressionVisitor().visit(node.test)
yield ") {"
for child in node.body:
yield from self.visit(child)
yield "}"
if node.orelse:
raise NotImplementedError(node, "orelse")
def visit_lvalue(self, lvalue: ast.expr) -> Iterable[str]:
if isinstance(lvalue, ast.Tuple):
yield f"std::tie({', '.join(flatmap(ExpressionVisitor().visit, lvalue.elts))})"
elif isinstance(lvalue, (ast.Name, ast.Subscript)):
yield from ExpressionVisitor().visit(lvalue)
else:
raise NotImplementedError(lvalue)
def visit_Assign(self, node: ast.Assign) -> Iterable[str]:
if len(node.targets) != 1:
raise NotImplementedError(node)
yield from self.visit_lvalue(node.targets[0])
yield " = "
yield from ExpressionVisitor().visit(node.value)
yield ";"
def visit_For(self, node: ast.For) -> Iterable[str]:
if not isinstance(node.target, ast.Name):
raise NotImplementedError(node)
yield f"for (auto {node.target.id} : "
yield from ExpressionVisitor().visit(node.iter)
yield ") {"
for child in node.body:
yield from self.visit(child)
yield "}"
if node.orelse:
raise NotImplementedError(node, "orelse")
# coding: utf-8
import subprocess
import clang_format
clang = clang_format._get_executable("clang-format") # noqa
def format_code(code: str) -> str:
return subprocess.check_output([clang, "-style=LLVM"], input=code.encode("utf-8")).decode("utf-8")
from typing import Callable, TypeVar
T = TypeVar("T")
def fork(_f: Callable[[], T]) -> T:
# stub
pass
def sync():
# stub
pass
def is_cpp():
return False
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment