add docstring support for modules and functions (both user written and...

add docstring support for modules and functions (both user written and builtin).  add doc strings for `__builtin__` and `imp` modules.

src/capi/modsupport.cpp

@@ -380,7 +380,7 @@ extern "C" PyObject* Py_BuildValue(const char* fmt, ...) noexcept {
 
 extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, const char* doc, PyObject* self,
                                     int apiver) noexcept {
-    BoxedModule* module = createModule(name, "__builtin__");
+    BoxedModule* module = createModule(name, "__builtin__", doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None
     Box* passthrough = static_cast<Box*>(self);
@@ -396,10 +396,6 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
         methods++;
     }
 
-    if (doc) {
-        module->setattr("__doc__", boxStrConstant(doc), NULL);
-    }
-
     return module;
 }
 

src/codegen/ast_interpreter.cpp

@@ -675,7 +675,6 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
         }
         assert(closure);
     }
-
     return boxCLFunction(cl, closure, is_generator, u.il);
 }
 

src/codegen/irgen/hooks.cpp

@@ -101,6 +101,17 @@ const std::string SourceInfo::getName() {
     }
 }
 
+Box* SourceInfo::getDocString() {
+    AST_Str* first_str = NULL;
+
+    if (body.size() > 0 && body[0]->type == AST_TYPE::Expr
+        && static_cast<AST_Expr*>(body[0])->value->type == AST_TYPE::Str) {
+        return boxString(static_cast<AST_Str*>(static_cast<AST_Expr*>(body[0])->value)->str_data);
+    }
+
+    return None;
+}
+
 ScopeInfo* SourceInfo::getScopeInfo() {
     return scoping->getScopeInfoForNode(ast);
 }
@@ -297,6 +308,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         SourceInfo* si = new SourceInfo(bm, scoping, m, m->body);
         CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
 
+        bm->setattr("__doc__", si->getDocString(), NULL);
+
         EffortLevel effort = initialEffort();
 
         cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL);

src/core/types.h

@@ -261,6 +261,8 @@ public:
     const std::string getName();
     InternedString mangleName(InternedString id);
 
+    Box* getDocString();
+
     SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body);
 };
 
@@ -515,7 +517,7 @@ class BoxedClass;
 void setupRuntime();
 void teardownRuntime();
 BoxedModule* createAndRunModule(const std::string& name, const std::string& fn);
-BoxedModule* createModule(const std::string& name, const std::string& fn);
+BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc = NULL);
 
 // TODO where to put this
 void appendToSysPath(const std::string& path);

src/runtime/builtin_modules/builtins.cpp

@@ -981,7 +981,9 @@ Box* pydumpAddr(Box* p) {
 }
 
 void setupBuiltins() {
-    builtins_module = createModule("__builtin__", "__builtin__");
+    builtins_module = createModule("__builtin__", "__builtin__",
+                                   "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
+                                   "the `nil' object; Ellipsis represents `...' in slices.");
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");

src/runtime/import.cpp

@@ -684,7 +684,9 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
 }
 
 void setupImport() {
-    BoxedModule* imp_module = createModule("imp", "__builtin__");
+    BoxedModule* imp_module
+        = createModule("imp", "__builtin__", "'This module provides the components needed to build your own\n"
+                                             "__import__ function.  Undocumented functions are obsolete.'");
 
     imp_module->giveAttr("PY_SOURCE", boxInt(SearchResult::PY_SOURCE));
     imp_module->giveAttr("PY_COMPILED", boxInt(SearchResult::PY_COMPILED));

src/runtime/types.cpp

@@ -260,22 +260,22 @@ std::string builtinStr("__builtin__");
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f)
     : in_weakreflist(NULL), f(f), closure(NULL), isGenerator(false), ndefaults(0), defaults(NULL), modname(NULL),
-      name(NULL) {
+      name(NULL), doc(NULL) {
     if (f->source) {
         this->modname = f->source->parent_module->getattr("__name__", NULL);
+        this->doc = f->source->getDocString();
     } else {
         this->modname = boxStringPtr(&builtinStr);
+        this->doc = None;
     }
 
-    this->giveAttr("__doc__", None);
-
     assert(f->num_defaults == ndefaults);
 }
 
 extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults,
                                                 BoxedClosure* closure, bool isGenerator)
     : in_weakreflist(NULL), f(f), closure(closure), isGenerator(isGenerator), ndefaults(0), defaults(NULL),
-      modname(NULL), name(NULL) {
+      modname(NULL), name(NULL), doc(NULL) {
     if (defaults.size()) {
         // make sure to initialize defaults first, since the GC behavior is triggered by ndefaults,
         // and a GC can happen within this constructor:
@@ -286,8 +286,10 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_
 
     if (f->source) {
         this->modname = f->source->parent_module->getattr("__name__", NULL);
+        this->doc = f->source->getDocString();
     } else {
         this->modname = boxStringPtr(&builtinStr);
+        this->doc = None;
     }
 
     assert(f->num_defaults == ndefaults);
@@ -306,21 +308,22 @@ BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults
     if (f->source) {
         this->name = static_cast<BoxedString*>(boxString(f->source->getName()));
     }
-
-    this->giveAttr("__doc__", None);
 }
 
-BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name)
+BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc)
     : BoxedBuiltinFunctionOrMethod(f, name, {}) {
+
+    this->doc = doc ? boxStrConstant(doc) : None;
 }
 
 BoxedBuiltinFunctionOrMethod::BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name,
                                                            std::initializer_list<Box*> defaults, BoxedClosure* closure,
-                                                           bool isGenerator)
+                                                           bool isGenerator, const char* doc)
     : BoxedFunctionBase(f, defaults, closure, isGenerator) {
 
     assert(name);
     this->name = static_cast<BoxedString*>(boxString(name));
+    this->doc = doc ? boxStrConstant(doc) : None;
 }
 
 extern "C" void functionGCHandler(GCVisitor* v, Box* b) {
@@ -335,6 +338,9 @@ extern "C" void functionGCHandler(GCVisitor* v, Box* b) {
     if (f->modname)
         v->visit(f->modname);
 
+    if (f->doc)
+        v->visit(f->doc);
+
     if (f->closure)
         v->visit(f->closure);
 
@@ -357,9 +363,10 @@ static void functionDtor(Box* b) {
     self->dependent_ics.~ICInvalidator();
 }
 
-BoxedModule::BoxedModule(const std::string& name, const std::string& fn) : fn(fn) {
+BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) : fn(fn) {
     this->giveAttr("__name__", boxString(name));
     this->giveAttr("__file__", boxString(fn));
+    this->giveAttr("__doc__", doc ? boxStrConstant(doc) : None);
 }
 
 std::string BoxedModule::name() {
@@ -926,6 +933,7 @@ static void typeSetModule(Box* _type, PyObject* value, void* context) {
     type->setattr("__module__", value, NULL);
 }
 
+
 Box* typeHash(BoxedClass* self) {
     assert(isSubclass(self->cls, type_cls));
 
@@ -1878,6 +1886,8 @@ void setupRuntime() {
     function_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)functionRepr, STR, 1)));
     function_cls->giveAttr("__module__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT,
                                                                    offsetof(BoxedFunction, modname), false));
+    function_cls->giveAttr(
+        "__doc__", new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedFunction, doc), false));
     function_cls->giveAttr("__get__", new BoxedFunction(boxRTFunction((void*)functionGet, UNKNOWN, 3)));
     function_cls->giveAttr("__call__",
                            new BoxedFunction(boxRTFunction((void*)functionCall, UNKNOWN, 1, 0, true, true)));
@@ -1891,6 +1901,9 @@ void setupRuntime() {
         "__repr__", new BoxedFunction(boxRTFunction((void*)builtinFunctionOrMethodRepr, STR, 1)));
     builtin_function_or_method_cls->giveAttr(
         "__name__", new (pyston_getset_cls) BoxedGetsetDescriptor(builtinFunctionOrMethodName, NULL, NULL));
+    builtin_function_or_method_cls->giveAttr(
+        "__doc__",
+        new BoxedMemberDescriptor(BoxedMemberDescriptor::OBJECT, offsetof(BoxedBuiltinFunctionOrMethod, doc), false));
     builtin_function_or_method_cls->freeze();
 
     instancemethod_cls->giveAttr(
@@ -2017,16 +2030,15 @@ void setupRuntime() {
     TRACK_ALLOCATIONS = true;
 }
 
-BoxedModule* createModule(const std::string& name, const std::string& fn) {
+BoxedModule* createModule(const std::string& name, const std::string& fn, const char* doc) {
     assert(fn.size() && "probably wanted to set the fn to <stdin>?");
-    BoxedModule* module = new BoxedModule(name, fn);
+    BoxedModule* module = new BoxedModule(name, fn, doc);
 
     BoxedDict* d = getSysModulesDict();
     Box* b_name = boxStringPtr(&name);
     ASSERT(d->d.count(b_name) == 0, "%s", name.c_str());
     d->d[b_name] = module;
 
-    module->giveAttr("__doc__", None);
     return module;
 }
 

src/runtime/types.h

@@ -442,6 +442,7 @@ public:
     // Accessed via member descriptor
     Box* modname;      // __module__
     BoxedString* name; // __name__ (should be here or in one of the derived classes?)
+    Box* doc;          // __doc__
 
     BoxedFunctionBase(CLFunction* f);
     BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
@@ -461,9 +462,9 @@ public:
 
 class BoxedBuiltinFunctionOrMethod : public BoxedFunctionBase {
 public:
-    BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name);
+    BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, const char* doc = NULL);
     BoxedBuiltinFunctionOrMethod(CLFunction* f, const char* name, std::initializer_list<Box*> defaults,
-                                 BoxedClosure* closure = NULL, bool isGenerator = false);
+                                 BoxedClosure* closure = NULL, bool isGenerator = false, const char* doc = NULL);
 
     DEFAULT_CLASS(builtin_function_or_method_cls);
 };
@@ -476,7 +477,7 @@ public:
     std::string fn;
     FutureFlags future_flags;
 
-    BoxedModule(const std::string& name, const std::string& fn);
+    BoxedModule(const std::string& name, const std::string& fn, const char* doc = NULL);
     std::string name();
 
     DEFAULT_CLASS(module_cls);

test/tests/docstrings.py

@@ -2,6 +2,19 @@ print __doc__
 __doc__ = "module_doc"
 print __doc__
 
+import test_package
+print test_package.__doc__
+test_package.__doc__ = "changeable module docs"
+print test_package.__doc__
+
+def foo():
+    """ foo docs go here """
+    pass
+
+print foo.__doc__
+foo.__doc__ = "no they don't"
+print foo.__doc__
+
 class C1(object):
     print 1, __doc__
     "hello world"

test/tests/test_package/__init__.py

+"""
+  module docs go here
+"""