From c6a9c223be9265f72b042dfcf41aad7006e9f9ba Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 30 Sep 2019 12:03:05 -0700 Subject: [PATCH 001/102] Remove stale reference --- .../Impl/Microsoft.Python.LanguageServer.csproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index ca455737f..a33882e0d 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -33,14 +33,6 @@ - - - $(AnalysisReference)/Microsoft.Python.Analysis.Engine.dll - - - PreserveNewest - - From 745a3eb553e03542a21c0d4b3b8f42d4d06c0d56 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 24 Oct 2019 22:30:51 -0700 Subject: [PATCH 002/102] Fix saving of ALL import statements --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 2 +- .../Ast/Impl/Dependencies/DependencyWalker.cs | 1 - .../Resolution/ModuleResolutionBase.cs | 1 - src/Parsing/Impl/Ast/PythonAst.cs | 22 ++++++++++++------ src/Parsing/Impl/Ast/SuiteStatement.cs | 23 +++++++++++++++---- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 90cbb451b..dd1a020c5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -416,7 +416,7 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode x is ImportStatement || x is FromImportStatement); + ast.ReduceToImports(); document.SetAst(ast); var eval = new ExpressionEval(walker.Eval.Services, document, ast); diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs index 2f2270251..f0ef93f98 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 184b2027c..40aa60020 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -26,7 +26,6 @@ using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; -using Microsoft.Python.Core.Services; namespace Microsoft.Python.Analysis.Modules.Resolution { internal abstract class ModuleResolutionBase { diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 5ebeee6e7..58c99e8c4 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -49,12 +49,14 @@ public PythonAst(IEnumerable existingAst) { locs.AddRange(a.NewLineLocations.Select(ll => new NewLineLocation(ll.EndIndex + offset, ll.Kind))); offset = locs.LastOrDefault().EndIndex; } + NewLineLocations = locs.ToArray(); offset = 0; foreach (var a in asts) { comments.AddRange(a.CommentLocations.Select(cl => new SourceLocation(cl.Line + offset, cl.Column))); offset += a.NewLineLocations.Length + 1; } + CommentLocations = comments.ToArray(); } @@ -74,12 +76,13 @@ public PythonAst(IEnumerable existingAst) { /// public bool HasVerbatim { get; internal set; } - public override IEnumerable GetChildNodes() => new[] { _body }; + public override IEnumerable GetChildNodes() => new[] {_body}; public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { _body.Walk(walker); } + walker.PostWalk(this); } @@ -87,15 +90,16 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken if (await walker.WalkAsync(this, cancellationToken)) { await _body.WalkAsync(walker, cancellationToken); } + await walker.PostWalkAsync(this, cancellationToken); } public override Statement Body => _body; public PythonLanguageVersion LanguageVersion { get; } - public void Reduce(Func filter) { + public void ReduceToImports() { lock (_lock) { - (Body as SuiteStatement)?.FilterStatements(filter); + (Body as SuiteStatement)?.ReduceToImports(); _attributes?.Clear(); Variables?.Clear(); CommentLocations = Array.Empty(); @@ -121,6 +125,7 @@ public void SetAttribute(Node node, object key, object value) { if (!_attributes.TryGetValue(node, out var nodeAttrs)) { nodeAttrs = _attributes[node] = new Dictionary(); } + nodeAttrs[key] = value; } } @@ -141,8 +146,10 @@ internal void SetAttributes(Dictionary> attribu } #region ILocationConverter + public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index); public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex); + #endregion internal int GetLineEndFromPosition(int index) { @@ -150,6 +157,7 @@ internal int GetLineEndFromPosition(int index) { if (loc.Line >= NewLineLocations.Length) { return index; } + var res = NewLineLocations[loc.Line - 1]; switch (res.Kind) { case NewLineKind.LineFeed: @@ -164,8 +172,7 @@ internal int GetLineEndFromPosition(int index) { internal override bool ExposesLocalVariable(PythonVariable variable) => true; - internal override void FinishBind(PythonNameBinder binder) { - } + internal override void FinishBind(PythonNameBinder binder) { } internal override PythonVariable BindReference(PythonNameBinder binder, string name) => EnsureVariable(name); @@ -186,6 +193,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow return true; } } + variable = null; return false; } @@ -197,7 +205,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow /// for variables explicitly declared global by the user, and names accessed /// but not defined in the lexical scope. /// - internal PythonVariable/*!*/ EnsureGlobalVariable(string name) { + internal PythonVariable /*!*/ EnsureGlobalVariable(string name) { if (!TryGetVariable(name, out var variable)) { variable = CreateVariable(name, VariableKind.Global); } @@ -206,7 +214,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow } - internal PythonVariable/*!*/ EnsureNonlocalVariable(string name) { + internal PythonVariable /*!*/ EnsureNonlocalVariable(string name) { if (!TryGetVariable(name, out var variable)) { variable = CreateVariable(name, VariableKind.Nonlocal); } diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs index 920a1d8b6..25fd90d72 100644 --- a/src/Parsing/Impl/Ast/SuiteStatement.cs +++ b/src/Parsing/Impl/Ast/SuiteStatement.cs @@ -14,9 +14,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -33,8 +31,8 @@ public SuiteStatement(Statement[] statements) { public IList Statements => _statements; public override IEnumerable GetChildNodes() => _statements.WhereNotNull(); - public void FilterStatements(Func filter) - => _statements = _statements.Where(filter).ToArray(); + public void ReduceToImports() + => _statements = new FilteredWalker(this).Statements.ToArray(); public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { @@ -164,5 +162,22 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) { _statements[0].SetLeadingWhiteSpace(ast, whiteSpace); } } + + private sealed class FilteredWalker : PythonWalker { + public FilteredWalker(Node n) { + n.Walk(this); + } + + public List Statements { get; } = new List(); + + public override bool Walk(ImportStatement s) { + Statements.Add(s); + return false; + } + public override bool Walk(FromImportStatement s) { + Statements.Add(s); + return false; + } + } } } From 2a1ca1401d2002051aba20b074dd856ca6f92a83 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 25 Oct 2019 19:02:33 -0700 Subject: [PATCH 003/102] Attributes --- src/Caching/Impl/Models/FromImportModel.cs | 3 +++ src/Caching/Impl/Models/FunctionAttributes.cs | 1 + src/Caching/Impl/Models/GenericParameterValueModel.cs | 3 +++ src/Caching/Impl/Models/ImportModel.cs | 3 +++ src/Caching/Impl/Models/IndexSpanModel.cs | 2 ++ src/Caching/Impl/Models/NewLineModel.cs | 2 ++ src/Caching/Impl/Models/OverloadModel.cs | 3 +++ src/Caching/Impl/Models/ParameterModel.cs | 2 ++ 8 files changed, 19 insertions(+) diff --git a/src/Caching/Impl/Models/FromImportModel.cs b/src/Caching/Impl/Models/FromImportModel.cs index f5a670056..0e81b8b62 100644 --- a/src/Caching/Impl/Models/FromImportModel.cs +++ b/src/Caching/Impl/Models/FromImportModel.cs @@ -13,10 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; + namespace Microsoft.Python.Analysis.Caching.Models { /// /// Represents from import statement for dependency resolution. /// + [Serializable] internal sealed class FromImportModel { public string[] RootNames { get; set; } public string[] MemberNames { get; set; } diff --git a/src/Caching/Impl/Models/FunctionAttributes.cs b/src/Caching/Impl/Models/FunctionAttributes.cs index 1c0f3da74..45286ca07 100644 --- a/src/Caching/Impl/Models/FunctionAttributes.cs +++ b/src/Caching/Impl/Models/FunctionAttributes.cs @@ -17,6 +17,7 @@ namespace Microsoft.Python.Analysis.Caching.Models { [Flags] + [Serializable] internal enum FunctionAttributes { Normal, Abstract, diff --git a/src/Caching/Impl/Models/GenericParameterValueModel.cs b/src/Caching/Impl/Models/GenericParameterValueModel.cs index 17f0c68f6..c6287920d 100644 --- a/src/Caching/Impl/Models/GenericParameterValueModel.cs +++ b/src/Caching/Impl/Models/GenericParameterValueModel.cs @@ -13,11 +13,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; + namespace Microsoft.Python.Analysis.Caching.Models { /// /// Model for actual values assigned to generic parameters. /// I.e. if class is based on Generic[T], what is assigned to T. /// + [Serializable] internal sealed class GenericParameterValueModel { /// /// Generic parameter name as defined by TypeVar, such as T. diff --git a/src/Caching/Impl/Models/ImportModel.cs b/src/Caching/Impl/Models/ImportModel.cs index 42d479291..954fe7b56 100644 --- a/src/Caching/Impl/Models/ImportModel.cs +++ b/src/Caching/Impl/Models/ImportModel.cs @@ -13,10 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; + namespace Microsoft.Python.Analysis.Caching.Models { /// /// Represents import statement for dependency resolution. /// + [Serializable] internal sealed class ImportModel { public DottedNameModel[] ModuleNames { get; set; } public bool ForceAbsolute { get; set; } diff --git a/src/Caching/Impl/Models/IndexSpanModel.cs b/src/Caching/Impl/Models/IndexSpanModel.cs index b9ccc45d7..08f9719bb 100644 --- a/src/Caching/Impl/Models/IndexSpanModel.cs +++ b/src/Caching/Impl/Models/IndexSpanModel.cs @@ -14,11 +14,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Diagnostics; using Microsoft.Python.Core.Text; // ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] [DebuggerDisplay("{Start}:({Length})")] internal sealed class IndexSpanModel { public int Start { get; set; } diff --git a/src/Caching/Impl/Models/NewLineModel.cs b/src/Caching/Impl/Models/NewLineModel.cs index b3021102d..9b4fe0bf5 100644 --- a/src/Caching/Impl/Models/NewLineModel.cs +++ b/src/Caching/Impl/Models/NewLineModel.cs @@ -14,9 +14,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] internal sealed class NewLineModel { public int EndIndex { get; set; } public NewLineKind Kind { get; set; } diff --git a/src/Caching/Impl/Models/OverloadModel.cs b/src/Caching/Impl/Models/OverloadModel.cs index 1f0014eef..9e03da580 100644 --- a/src/Caching/Impl/Models/OverloadModel.cs +++ b/src/Caching/Impl/Models/OverloadModel.cs @@ -13,7 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; + namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] internal sealed class OverloadModel { public ParameterModel[] Parameters { get; set; } public string ReturnType { get; set; } diff --git a/src/Caching/Impl/Models/ParameterModel.cs b/src/Caching/Impl/Models/ParameterModel.cs index 6006b385b..63e627753 100644 --- a/src/Caching/Impl/Models/ParameterModel.cs +++ b/src/Caching/Impl/Models/ParameterModel.cs @@ -13,9 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching.Models { + [Serializable] internal sealed class ParameterModel { public string Name { get; set; } public string Type { get; set; } From 91f3ca6d085dd0a2f5ad8ab2a6de863440b3d10c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 25 Oct 2019 23:01:35 -0700 Subject: [PATCH 004/102] Initial --- .../Ast/Impl/Analyzer/AnalysisModuleKey.cs | 2 +- .../Definitions/IModuleDatabaseService.cs | 7 -- .../Ast/Impl/Modules/DependencyProvider.cs | 4 -- .../DependencyCollectorExtensions.cs | 41 ----------- src/Caching/Impl/Models/CallableModel.cs | 8 ++- src/Caching/Impl/Models/ClassModel.cs | 19 ++--- src/Caching/Impl/Models/DottedNameModel.cs | 23 ------ src/Caching/Impl/Models/FromImportModel.cs | 29 -------- src/Caching/Impl/Models/FunctionModel.cs | 13 ++-- src/Caching/Impl/Models/ImportModel.cs | 27 ------- src/Caching/Impl/Models/MemberModel.cs | 5 ++ src/Caching/Impl/Models/ModuleModel.cs | 71 +++---------------- src/Caching/Impl/Models/NamedTupleModel.cs | 3 +- src/Caching/Impl/Models/PropertyModel.cs | 5 +- src/Caching/Impl/Models/TypeVarModel.cs | 4 +- src/Caching/Impl/Models/VariableModel.cs | 12 ++-- src/Caching/Impl/ModuleDatabase.cs | 66 ++++------------- src/Caching/Impl/ModuleFactory.cs | 33 ++++++--- src/Caching/Impl/ModuleUniqueId.cs | 6 +- src/Caching/Impl/PythonDbModule.cs | 2 +- src/Caching/Impl/QualifiedNameParts.cs | 4 +- src/Caching/Impl/RestoredGlobalScope.cs | 11 +-- src/Caching/Impl/TypeNames.cs | 38 +++++----- src/Caching/Test/AnalysisCachingTestBase.cs | 17 ----- src/LanguageServer/Impl/Program.cs | 2 +- 25 files changed, 127 insertions(+), 325 deletions(-) delete mode 100644 src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs delete mode 100644 src/Caching/Impl/Models/DottedNameModel.cs delete mode 100644 src/Caching/Impl/Models/FromImportModel.cs delete mode 100644 src/Caching/Impl/Models/ImportModel.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index 11c9d148c..608f2b719 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Analyzer { [DebuggerDisplay("{Name} : {FilePath}")] - internal readonly struct AnalysisModuleKey : IEquatable { + public readonly struct AnalysisModuleKey : IEquatable { public string Name { get; } public string FilePath { get; } public bool IsTypeshed { get; } diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index fb40a5bfc..16c8a213e 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -28,13 +28,6 @@ internal interface IModuleDatabaseService: IModuleDatabaseCache { /// Python module global scope. bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs); - /// - /// Retrieves dependencies from the module persistent state. - /// - /// Python module to restore analysis for. - /// Python module dependency provider. - bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp); - /// /// Writes module data to the database. /// diff --git a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs index 1e0e4752a..63cb5aeb9 100644 --- a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs +++ b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs @@ -35,10 +35,6 @@ public DependencyProvider(IPythonModule module, IServiceContainer services) { #region IDependencyProvider public ISet GetDependencies(PythonAst ast) { - if (_dbService != null && _dbService.TryRestoreDependencies(_module, out var dp)) { - return dp.GetDependencies(ast); - } - // TODO: try and handle LoadFunctionDependencyModules functionality here. var dw = new DependencyWalker(_module, ast); return dw.Dependencies; diff --git a/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs b/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs deleted file mode 100644 index b200daf5c..000000000 --- a/src/Caching/Impl/Extensions/DependencyCollectorExtensions.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Dependencies; -using Microsoft.Python.Core.Collections; - -namespace Microsoft.Python.Analysis.Caching { - internal static class DependencyCollectorExtensions { - public static void AddImports(this DependencyCollector dc, IEnumerable imports) { - foreach (var imp in imports) { - foreach (var dottedName in imp.ModuleNames) { - var importNames = ImmutableArray.Empty; - foreach (var part in dottedName.NameParts) { - importNames = importNames.Add(part); - dc.AddImport(importNames, imp.ForceAbsolute); - } - } - } - } - - public static void AddFromImports(this DependencyCollector dc, IEnumerable imports) { - foreach (var imp in imports) { - dc.AddFromImport(imp.RootNames, imp.MemberNames, imp.DotCount, imp.ForceAbsolute); - } - } - } -} diff --git a/src/Caching/Impl/Models/CallableModel.cs b/src/Caching/Impl/Models/CallableModel.cs index 323d5646f..1bf169514 100644 --- a/src/Caching/Impl/Models/CallableModel.cs +++ b/src/Caching/Impl/Models/CallableModel.cs @@ -32,9 +32,10 @@ internal abstract class CallableModel : MemberModel { [NonSerialized] private readonly ReentrancyGuard _processing = new ReentrancyGuard(); + protected CallableModel() { } // For de-serializer from JSON - protected CallableModel(IPythonType callable) { + protected CallableModel(IPythonType callable, IServiceContainer services) { var functions = new List(); var classes = new List(); @@ -50,10 +51,10 @@ protected CallableModel(IPythonType callable) { case IPythonFunctionType ft1 when ft1.IsLambda(): break; case IPythonFunctionType ft2: - functions.Add(new FunctionModel(ft2)); + functions.Add(new FunctionModel(ft2, services)); break; case IPythonClassType cls: - classes.Add(new ClassModel(cls)); + classes.Add(new ClassModel(cls, services)); break; } } @@ -61,6 +62,7 @@ protected CallableModel(IPythonType callable) { Id = callable.Name.GetStableHash(); Name = callable.Name; + DeclaringModuleId = callable.DeclaringModule.GetUniqueId(services, AnalysisCachingLevel.Library); QualifiedName = callable.QualifiedName; Documentation = callable.Documentation; Classes = classes.ToArray(); diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index 527816157..16582e247 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -53,7 +53,7 @@ internal sealed class ClassModel : MemberModel { public ClassModel() { } // For de-serializer from JSON - public ClassModel(IPythonClassType cls) { + public ClassModel(IPythonClassType cls, IServiceContainer services) { var methods = new List(); var properties = new List(); var fields = new List(); @@ -78,21 +78,21 @@ public ClassModel(IPythonClassType cls) { if (!ct.DeclaringModule.Equals(cls.DeclaringModule)) { continue; } - innerClasses.Add(new ClassModel(ct)); + innerClasses.Add(new ClassModel(ct, services)); break; case IPythonFunctionType ft when ft.IsLambda(): break; case IPythonFunctionType ft when ft.Name == name: - methods.Add(new FunctionModel(ft)); + methods.Add(new FunctionModel(ft, services)); break; case IPythonPropertyType prop when prop.Name == name: - properties.Add(new PropertyModel(prop)); + properties.Add(new PropertyModel(prop, services)); break; case IPythonInstance inst: - fields.Add(VariableModel.FromInstance(name, inst)); + fields.Add(VariableModel.FromInstance(name, inst, services)); break; case IPythonType t: - fields.Add(VariableModel.FromType(name, t)); + fields.Add(VariableModel.FromType(name, t, services)); break; } } @@ -100,6 +100,7 @@ public ClassModel(IPythonClassType cls) { Name = cls.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : cls.Name; Id = Name.GetStableHash(); + DeclaringModuleId = cls.DeclaringModule.GetUniqueId(services); QualifiedName = cls.QualifiedName; IndexSpan = cls.Location.IndexSpan.ToModel(); @@ -107,9 +108,9 @@ public ClassModel(IPythonClassType cls) { Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null; var ntBases = cls.Bases.OfType().ToArray(); - NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b)).ToArray(); + NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b, services)).ToArray(); - Bases = cls.Bases.Except(ntBases).Select(t => t.GetPersistentQualifiedName()).ToArray(); + Bases = cls.Bases.Except(ntBases).Select(t => t.GetPersistentQualifiedName(services)).ToArray(); Methods = methods.ToArray(); Properties = properties.ToArray(); Fields = fields.ToArray(); @@ -125,7 +126,7 @@ public ClassModel(IPythonClassType cls) { GenericBaseParameters = GenericBaseParameters ?? Array.Empty(); GenericParameterValues = cls.GenericParameters - .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName() }) + .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName(services) }) .ToArray(); } diff --git a/src/Caching/Impl/Models/DottedNameModel.cs b/src/Caching/Impl/Models/DottedNameModel.cs deleted file mode 100644 index 1bbc05822..000000000 --- a/src/Caching/Impl/Models/DottedNameModel.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.Python.Analysis.Caching.Models { - [Serializable] - internal sealed class DottedNameModel { - public string[] NameParts { get; set; } - } -} diff --git a/src/Caching/Impl/Models/FromImportModel.cs b/src/Caching/Impl/Models/FromImportModel.cs deleted file mode 100644 index 0e81b8b62..000000000 --- a/src/Caching/Impl/Models/FromImportModel.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.Python.Analysis.Caching.Models { - /// - /// Represents from import statement for dependency resolution. - /// - [Serializable] - internal sealed class FromImportModel { - public string[] RootNames { get; set; } - public string[] MemberNames { get; set; } - public int DotCount { get; set; } - public bool ForceAbsolute { get; set; } - } -} diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index dbfd63ddf..2c365e6c1 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -18,6 +18,7 @@ using System.Linq; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -31,8 +32,8 @@ public FunctionModel() { } // For de-serializer from JSON [NonSerialized] private PythonFunctionType _function; - public FunctionModel(IPythonFunctionType func) : base(func) { - Overloads = func.Overloads.Select(FromOverload).ToArray(); + public FunctionModel(IPythonFunctionType func, IServiceContainer services) : base(func, services) { + Overloads = func.Overloads.Select(s => FromOverload(s, services)).ToArray(); } public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) @@ -61,15 +62,15 @@ public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlob private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); - private static OverloadModel FromOverload(IPythonFunctionOverload o) + private static OverloadModel FromOverload(IPythonFunctionOverload o, IServiceContainer services) => new OverloadModel { Parameters = o.Parameters.Select(p => new ParameterModel { Name = p.Name, - Type = p.Type.GetPersistentQualifiedName(), + Type = p.Type.GetPersistentQualifiedName(services), Kind = p.Kind, - DefaultValue = p.DefaultValue.GetPersistentQualifiedName(), + DefaultValue = p.DefaultValue.GetPersistentQualifiedName(services), }).ToArray(), - ReturnType = o.StaticReturnValue.GetPersistentQualifiedName() + ReturnType = o.StaticReturnValue.GetPersistentQualifiedName(services) }; } } diff --git a/src/Caching/Impl/Models/ImportModel.cs b/src/Caching/Impl/Models/ImportModel.cs deleted file mode 100644 index 954fe7b56..000000000 --- a/src/Caching/Impl/Models/ImportModel.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; - -namespace Microsoft.Python.Analysis.Caching.Models { - /// - /// Represents import statement for dependency resolution. - /// - [Serializable] - internal sealed class ImportModel { - public DottedNameModel[] ModuleNames { get; set; } - public bool ForceAbsolute { get; set; } - } -} diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index e7ce43282..94779f915 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -34,6 +34,11 @@ internal abstract class MemberModel { /// public string Name { get; set; } + /// + /// Unique id of declaring module. + /// + public string DeclaringModuleId { get; set; } + /// /// Member qualified name within the module, such as A.B.C. /// diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index fbca1705f..e1eae7b98 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -21,7 +21,6 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; -using Microsoft.Python.Parsing.Ast; // ReSharper disable MemberCanBePrivate.Global namespace Microsoft.Python.Analysis.Caching.Models { @@ -31,7 +30,7 @@ internal sealed class ModuleModel : MemberModel { /// Module unique id that includes version. /// public string UniqueId { get; set; } - + public string FilePath { get; set; } public string Documentation { get; set; } public FunctionModel[] Functions { get; set; } public VariableModel[] Variables { get; set; } @@ -50,11 +49,6 @@ internal sealed class ModuleModel : MemberModel { /// public int FileSize { get; set; } - public ImportModel[] Imports { get; set; } - public FromImportModel[] FromImports { get; set; } - public ImportModel[] StubImports { get; set; } - public FromImportModel[] StubFromImports { get; set; } - [NonSerialized] private Dictionary _modelCache; public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { @@ -80,13 +74,13 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta v.Source == VariableSource.Generic)) { if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { - typeVars[v.Name] = TypeVarModel.FromGeneric(v); + typeVars[v.Name] = TypeVarModel.FromGeneric(v, services); continue; } switch (v.Value) { case ITypingNamedTupleType nt: - namedTuples[nt.Name] = new NamedTupleModel(nt); + namedTuples[nt.Name] = new NamedTupleModel(nt, services); continue; case IPythonFunctionType ft when ft.IsLambda(): // No need to persist lambdas. @@ -97,7 +91,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta // x = type(func) break; case IPythonFunctionType ft: - var fm = GetFunctionModel(analysis, v, ft); + var fm = GetFunctionModel(analysis, v, ft, services); if (fm != null && !functions.ContainsKey(ft.Name)) { functions[ft.Name] = fm; continue; @@ -109,7 +103,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta case IPythonClassType cls when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals(analysis.Document.Stub): if (!classes.ContainsKey(cls.Name)) { - classes[cls.Name] = new ClassModel(cls); + classes[cls.Name] = new ClassModel(cls, services); continue; } break; @@ -117,24 +111,16 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals // Do not re-declare classes and functions as variables in the model. if (!variables.ContainsKey(v.Name)) { - variables[v.Name] = VariableModel.FromVariable(v); + variables[v.Name] = VariableModel.FromVariable(v, services); } } - // Take dependencies from imports. If module has stub we should also take - // dependencies from there since persistent state is based on types that - // are combination of stub and the module. Sometimes stub may import more - // and we must make sure dependencies are restored before the module. - var primaryDependencyWalker = new DependencyWalker(analysis.Ast); - var stubDependencyWalker = analysis.Document.Stub != null ? new DependencyWalker(analysis.Document.Stub.Analysis.Ast) : null; - var stubImports = stubDependencyWalker?.Imports ?? Enumerable.Empty(); - var stubFromImports = stubDependencyWalker?.FromImports ?? Enumerable.Empty(); - return new ModuleModel { Id = uniqueId.GetStableHash(), UniqueId = uniqueId, Name = analysis.Document.Name, QualifiedName = analysis.Document.QualifiedName, + FilePath = analysis.Document.FilePath, Documentation = analysis.Document.Documentation, Functions = functions.Values.ToArray(), Variables = variables.Values.ToArray(), @@ -145,26 +131,22 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals EndIndex = l.EndIndex, Kind = l.Kind }).ToArray(), - FileSize = analysis.Ast.EndIndex, - Imports = primaryDependencyWalker.Imports.ToArray(), - FromImports = primaryDependencyWalker.FromImports.ToArray(), - StubImports = stubImports.ToArray(), - StubFromImports = stubFromImports.ToArray() + FileSize = analysis.Ast.EndIndex }; } - private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVariable v, IPythonFunctionType f) { + private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVariable v, IPythonFunctionType f, IServiceContainer services) { if (v.Source == VariableSource.Import && !f.DeclaringModule.Equals(analysis.Document) && !f.DeclaringModule.Equals(analysis.Document.Stub)) { // It may be that the function is from a child module via import. // For example, a number of functions in 'os' are imported from 'nt' on Windows via // star import. Their stubs, however, come from 'os' stub. The function then have declaring // module as 'nt' rather than 'os' and 'nt' does not have a stub. In this case use function // model like if function was declared in 'os'. - return new FunctionModel(f); + return new FunctionModel(f, services); } if (f.DeclaringModule.Equals(analysis.Document) || f.DeclaringModule.Equals(analysis.Document.Stub)) { - return new FunctionModel(f); + return new FunctionModel(f, services); } return null; } @@ -183,36 +165,5 @@ public override MemberModel GetModel(string name) { public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); - - private sealed class DependencyWalker : PythonWalker { - public List Imports { get; } = new List(); - public List FromImports { get; } = new List(); - - public DependencyWalker(PythonAst ast) { - ast.Walk(this); - } - - public override bool Walk(ImportStatement import) { - var model = new ImportModel { - ForceAbsolute = import.ForceAbsolute, - ModuleNames = import.Names.Select(mn => new DottedNameModel { - NameParts = mn.Names.Select(nex => nex.Name).ToArray() - }).ToArray() - }; - Imports.Add(model); - return false; - } - - public override bool Walk(FromImportStatement fromImport) { - var model = new FromImportModel { - ForceAbsolute = fromImport.ForceAbsolute, - RootNames = fromImport.Root.Names.Select(n => n.Name).ToArray(), - MemberNames = fromImport.Names.Select(n => n.Name).ToArray(), - DotCount = fromImport.Root is RelativeModuleName rn ? rn.DotCount : 0 - }; - FromImports.Add(model); - return false; - } - } } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index 82fed074b..88591b86f 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -33,9 +33,10 @@ internal sealed class NamedTupleModel: MemberModel { public NamedTupleModel() { } // For de-serializer from JSON - public NamedTupleModel(ITypingNamedTupleType nt) { + public NamedTupleModel(ITypingNamedTupleType nt, IServiceContainer services) { Id = nt.Name.GetStableHash(); Name = nt.Name; + DeclaringModuleId = nt.DeclaringModule.GetUniqueId(services, AnalysisCachingLevel.Library); QualifiedName = nt.QualifiedName; IndexSpan = nt.Location.IndexSpan.ToModel(); ItemNames = nt.ItemNames.ToArray(); diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index bea252b81..676dd6a7d 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -16,6 +16,7 @@ using System; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -28,8 +29,8 @@ public PropertyModel() { } // For de-serializer from JSON [NonSerialized] private PythonPropertyType _property; - public PropertyModel(IPythonPropertyType prop) : base(prop) { - ReturnType = prop.ReturnType.GetPersistentQualifiedName(); + public PropertyModel(IPythonPropertyType prop, IServiceContainer services) : base(prop, services) { + ReturnType = prop.ReturnType.GetPersistentQualifiedName(services); } public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index e630d801f..fd38dee3d 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -32,13 +32,13 @@ internal sealed class TypeVarModel : MemberModel { public object Covariant { get; set; } public object Contravariant { get; set; } - public static TypeVarModel FromGeneric(IVariable v) { + public static TypeVarModel FromGeneric(IVariable v, IServiceContainer services) { var g = (IGenericTypeParameter)v.Value; return new TypeVarModel { Id = g.Name.GetStableHash(), Name = g.Name, QualifiedName = g.QualifiedName, - Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName()).ToArray(), + Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName(services)).ToArray(), Bound = g.Bound?.QualifiedName, Covariant = g.Covariant, Contravariant = g.Contravariant diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index 892dac8ac..0f37ad7b6 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -26,27 +26,27 @@ namespace Microsoft.Python.Analysis.Caching.Models { internal sealed class VariableModel : MemberModel { public string Value { get; set; } - public static VariableModel FromVariable(IVariable v) => new VariableModel { + public static VariableModel FromVariable(IVariable v, IServiceContainer services) => new VariableModel { Id = v.Name.GetStableHash(), Name = v.Name, QualifiedName = v.Name, IndexSpan = v.Location.IndexSpan.ToModel(), - Value = v.Value.GetPersistentQualifiedName() + Value = v.Value.GetPersistentQualifiedName(services) }; - public static VariableModel FromInstance(string name, IPythonInstance inst) => new VariableModel { + public static VariableModel FromInstance(string name, IPythonInstance inst, IServiceContainer services) => new VariableModel { Id = name.GetStableHash(), Name = name, QualifiedName = name, - Value = inst.GetPersistentQualifiedName() + Value = inst.GetPersistentQualifiedName(services) }; - public static VariableModel FromType(string name, IPythonType t) => new VariableModel { + public static VariableModel FromType(string name, IPythonType t, IServiceContainer services) => new VariableModel { Id = name.GetStableHash(), Name = name, QualifiedName = name, IndexSpan = t.Location.IndexSpan.ToModel(), - Value = t.GetPersistentQualifiedName() + Value = t.GetPersistentQualifiedName(services) }; public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 75f423834..788ddc38a 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -14,25 +14,21 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using LiteDB; -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { internal sealed class ModuleDatabase : IModuleDatabaseService { - private readonly Dictionary _dependencies = new Dictionary(); private readonly object _lock = new object(); private readonly IServiceContainer _services; @@ -52,31 +48,6 @@ public ModuleDatabase(IServiceContainer services) { public int DatabaseFormatVersion => 2; public string CacheFolder { get; } - /// - /// Retrieves dependencies from the module persistent state. - /// - /// Python module to restore analysis for. - /// Python module dependency provider. - public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp) { - dp = null; - - if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { - return false; - } - - lock (_lock) { - if (_dependencies.TryGetValue(module.Name, out dp)) { - return true; - } - if (FindModuleModel(module.Name, module.FilePath, out var model)) { - dp = new DependencyProvider(module, model); - _dependencies[module.Name] = dp; - return true; - } - } - return false; - } - /// /// Creates global scope from module persistent state. /// Global scope is then can be used to construct module analysis. @@ -91,8 +62,8 @@ public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope } lock (_lock) { - if (FindModuleModel(module.Name, module.FilePath, out var model)) { - gs = new RestoredGlobalScope(model, module); + if (FindModuleModelByPath(module.Name, module.FilePath, out var model)) { + gs = new RestoredGlobalScope(model, module, this, _services); } } @@ -173,12 +144,11 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c /// module content (typically file sizes). /// private string FindDatabaseFile(string moduleName, string filePath) { - var interpreter = _services.GetService(); var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel()); - if (string.IsNullOrEmpty(uniqueId)) { - return null; - } + return string.IsNullOrEmpty(uniqueId) ? null : FindDatabaseFile(uniqueId); + } + private string FindDatabaseFile(string uniqueId) { // Try module name as is. var dbPath = Path.Combine(CacheFolder, $"{uniqueId}.db"); if (_fs.FileExists(dbPath)) { @@ -187,6 +157,7 @@ private string FindDatabaseFile(string moduleName, string filePath) { // TODO: resolving to a different version can be an option // Try with the major.minor Python version. + var interpreter = _services.GetService(); var pythonVersion = interpreter.Configuration.Version; dbPath = Path.Combine(CacheFolder, $"{uniqueId}({pythonVersion.Major}.{pythonVersion.Minor}).db"); @@ -199,15 +170,19 @@ private string FindDatabaseFile(string moduleName, string filePath) { return _fs.FileExists(dbPath) ? dbPath : null; } - private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) { - model = null; + public bool FindModuleModelByPath(string moduleName, string filePath, out ModuleModel model) + => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, filePath), out model); + + public bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) + => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, uniqueId), out model); + private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel model) { + model = null; // We don't cache results here. Module resolution service decides when to call in here // and it is responsible of overall management of the loaded Python modules. for (var retries = 50; retries > 0; --retries) { try { // TODO: make combined db rather than per module? - var dbPath = FindDatabaseFile(moduleName, filePath); if (string.IsNullOrEmpty(dbPath)) { return false; } @@ -229,18 +204,5 @@ private bool FindModuleModel(string moduleName, string filePath, out ModuleModel } private AnalysisCachingLevel GetCachingLevel() => _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; - - private sealed class DependencyProvider : IDependencyProvider { - private readonly ISet _dependencies; - - public DependencyProvider(IPythonModule module, ModuleModel model) { - var dc = new DependencyCollector(module); - dc.AddImports(model.Imports); - dc.AddFromImports(model.FromImports); - _dependencies = dc.Dependencies; - } - - public ISet GetDependencies(PythonAst ast) => _dependencies; - } } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 542b56da7..3bdf9a264 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -34,17 +35,24 @@ internal sealed class ModuleFactory { /// For use in tests so missing members will assert. internal static bool EnableMissingMemberAssertions { get; set; } + private static readonly ConcurrentDictionary _modulesCache + = new ConcurrentDictionary(); + // TODO: better resolve circular references. private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard(); private readonly ModuleModel _model; private readonly IGlobalScope _gs; + private readonly ModuleDatabase _db; + private readonly IServiceContainer _services; public IPythonModule Module { get; } public Location DefaultLocation { get; } - public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs) { + public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs, ModuleDatabase db, IServiceContainer services) { _model = model; _gs = gs; + _db = db; + _services = services; Module = module; DefaultLocation = new Location(Module); } @@ -100,7 +108,7 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { } var nextModel = currentModel.GetModel(memberName); - Debug.Assert(nextModel != null, + Debug.Assert(nextModel != null, $"Unable to find {string.Join(".", memberNames)} in module {Module.Name}"); if (nextModel == null) { return null; @@ -132,22 +140,31 @@ private IPythonModule GetModule(QualifiedNameParts parts) { if (parts.ModuleName == Module.Name) { return Module; } + using (_moduleReentrancy.Push(parts.ModuleName, out var reentered)) { if (reentered) { return null; } + + var module = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); + if (module == null && parts.ModuleId != null) { + if (!_modulesCache.TryGetValue(parts.ModuleId, out var m)) { + if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, out var model)) { + _modulesCache[parts.ModuleId] = m = new PythonDbModule(model, model.FilePath, _services); + m.Construct(model); + module = m; + } + } + } + // Here we do not call GetOrLoad since modules references here must // either be loaded already since they were required to create // persistent state from analysis. Also, occasionally types come // from the stub and the main module was never loaded. This, for example, // happens with io which has member with mmap type coming from mmap // stub rather than the primary mmap module. - var m = parts.IsStub - ? Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName) - : Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); - - if (m != null) { - return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(m) : m; + if (module != null) { + return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(module) : module; } return null; } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 51762ecfe..bc1da403f 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -26,10 +26,12 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { - public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel) + public static string GetUniqueId(this IPythonModule module, + IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); - public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel) { + public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, + IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { if(cachingLevel == AnalysisCachingLevel.None) { return null; } diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index 8da9634ed..bd3182e31 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -34,7 +34,7 @@ public PythonDbModule(ModuleModel model, string filePath, IServiceContainer serv } public void Construct(ModuleModel model) { - var gs = new RestoredGlobalScope(model, this); + var gs = new RestoredGlobalScope(model, this, Services.GetService(), Services); GlobalScope = gs; gs.ReconstructVariables(); } diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs index 4d1d33539..a4d27fae6 100644 --- a/src/Caching/Impl/QualifiedNameParts.cs +++ b/src/Caching/Impl/QualifiedNameParts.cs @@ -29,8 +29,8 @@ internal struct QualifiedNameParts { public ObjectType ObjectType; /// Module name. public string ModuleName; - /// Indicates if module is a stub. - public bool IsStub; + /// Module unique id. + public string ModuleId; /// Module member names such as 'A', 'B', 'C' from module:A.B.C. public IReadOnlyList MemberNames; } diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index 77e3bb255..b57292046 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -19,6 +19,7 @@ using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { @@ -27,12 +28,12 @@ internal sealed class RestoredGlobalScope : IRestoredGlobalScope { private ModuleModel _model; // Non-readonly b/c of DEBUG conditional. private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional. - public RestoredGlobalScope(ModuleModel model, IPythonModule module) { + public RestoredGlobalScope(ModuleModel model, IPythonModule module, ModuleDatabase db, IServiceContainer services) { _model = model ?? throw new ArgumentNullException(nameof(model)); Module = module ?? throw new ArgumentNullException(nameof(module)); Name = model.Name; - _factory = new ModuleFactory(_model, Module, this); - DeclareVariables(); + _factory = new ModuleFactory(_model, Module, this, db, services); + DeclareVariables(db, services); } public void ReconstructVariables() { @@ -47,11 +48,11 @@ public void ReconstructVariables() { #endif } - private void DeclareVariables() { + private void DeclareVariables(ModuleDatabase db, IServiceContainer services) { // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus first create members so we can find then, then populate them with content. - var mf = new ModuleFactory(_model, Module, this); + var mf = new ModuleFactory(_model, Module, this, db, services); // Generics first var typeVars = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs index 2d12f482a..5fcd4ce2b 100644 --- a/src/Caching/Impl/TypeNames.cs +++ b/src/Caching/Impl/TypeNames.cs @@ -19,6 +19,7 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching { internal static class TypeNames { @@ -26,27 +27,36 @@ internal static class TypeNames { /// Constructs persistent member name based on the member and the current module. /// Persistent name contains complete information for the member restoration code. /// - public static string GetPersistentQualifiedName(this IMember m) { + public static string GetPersistentQualifiedName(this IMember m, IServiceContainer services) { var t = m.GetPythonType(); + string name = null; if (!t.IsUnknown()) { switch (m) { case IPythonInstance _: // constants and strings map here. - return $"i:{t.QualifiedName}"; + name = $"i:{t.QualifiedName}"; + break; case IBuiltinsPythonModule b: return $"b:{b.QualifiedName}"; case PythonVariableModule vm: - return $"p:{vm.QualifiedName}"; + name = $"p:{vm.QualifiedName}"; + break; case IPythonModule mod: - return $"m:{mod.QualifiedName}"; + name = $"m:{mod.QualifiedName}"; + break; case IPythonType pt when pt.DeclaringModule.ModuleType == ModuleType.Builtins: return $"t:{(pt.TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : pt.QualifiedName)}"; case IPythonType pt: - return $"t:{pt.QualifiedName}"; + name = $"t:{pt.QualifiedName}"; + break; case null: break; } } - return null; + + if (name == null || t.DeclaringModule.ModuleType == ModuleType.Builtins) { + return name; + } + return $"{name}${t.DeclaringModule.GetUniqueId(services)}"; } /// @@ -61,6 +71,12 @@ public static bool DeconstructQualifiedName(string qualifiedName, out QualifiedN return false; } + var index = qualifiedName.IndexOf('$'); + if (index > 0) { + parts.ModuleId = qualifiedName.Substring(index + 1); + qualifiedName = qualifiedName.Substring(0, index); + } + GetObjectTypeFromPrefix(qualifiedName, ref parts, out var prefixOffset); GetModuleNameAndMembers(qualifiedName, ref parts, prefixOffset); @@ -102,7 +118,6 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN default: parts.ModuleName = typeName; parts.MemberNames = Array.Empty(); - DetermineModuleType(ref parts); break; } return; @@ -112,15 +127,6 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN parts.ModuleName = typeName.Substring(0, moduleSeparatorIndex); var memberNamesOffset = parts.ModuleName.Length + 1; parts.MemberNames = GetTypeNames(typeName.Substring(memberNamesOffset), '.'); - - DetermineModuleType(ref parts); - } - - private static void DetermineModuleType(ref QualifiedNameParts parts) { - if (parts.ModuleName.EndsWith("(stub)")) { - parts.ModuleName = parts.ModuleName.Substring(0, parts.ModuleName.Length - 6); - parts.IsStub = true; - } } public static IReadOnlyList GetTypeNames(string qualifiedTypeName, char separator) { diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 8baa49828..f81cb5657 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -66,23 +66,6 @@ internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonMod //var json = ToJson(model); //Baseline.CompareToFile(BaselineFileName, json); - // In real case dependency analysis will restore model dependencies. - // Here we don't go through the dependency analysis so we have to - // manually restore dependent modules. - var dc = new DependencyCollector(m); - dc.AddImports(model.Imports); - dc.AddFromImports(model.FromImports); - foreach(var dep in dc.Dependencies) { - m.Interpreter.ModuleResolution.GetOrLoadModule(dep.Name); - } - - var dcs = new DependencyCollector(m, true); - dcs.AddImports(model.StubImports); - dcs.AddFromImports(model.StubFromImports); - foreach (var dep in dcs.Dependencies) { - m.Interpreter.TypeshedResolution.GetOrLoadModule(dep.Name); - } - var analyzer = Services.GetService(); await analyzer.WaitForCompleteAnalysisAsync(); diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 024f9af18..24adfa924 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From b2e92ffd7d8cc98ea2a5ed103320d3d50628aee4 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 28 Oct 2019 11:23:07 -0700 Subject: [PATCH 005/102] Partial --- src/Caching/Impl/PythonLazyType.cs | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/Caching/Impl/PythonLazyType.cs diff --git a/src/Caching/Impl/PythonLazyType.cs b/src/Caching/Impl/PythonLazyType.cs new file mode 100644 index 000000000..d6e0deded --- /dev/null +++ b/src/Caching/Impl/PythonLazyType.cs @@ -0,0 +1,44 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Types; + +namespace Microsoft.Python.Analysis.Caching { + internal sealed class PythonLazyType: PythonTypeWrapper { + private readonly PythonDbModule _dbModule; + + public PythonLazyType(string name, string qualifiedTypeName, string documentation, PythonDbModule dbModule, BuiltinTypeId typeid, ModuleDatabase db) + : base(name, documentation, declaringModule, ) { + _dbModule = dbModule; + TypeId = typeid; + + } + + public override BuiltinTypeId TypeId { get; } + + public override IEnumerable GetMemberNames() { + return base.GetMemberNames(); + } + + public override IMember GetMember(string name) { + return base.GetMember(name); + } + + private void EnsureInnerType() { + _dbModule + } + } +} From 9b691e6bd1229891eb991d972a512f37e4b711ff Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 31 Oct 2019 17:51:53 -0700 Subject: [PATCH 006/102] Part I --- .../Impl/Analyzer/Definitions/IAnalyzable.cs | 7 -- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 22 +++++- .../Ast/Impl/Analyzer/PythonAnalyzerEntry.cs | 8 +- .../Impl/Analyzer/PythonAnalyzerSession.cs | 25 ------- .../Definitions/IModuleDatabaseService.cs | 16 ++-- .../Definitions/IRestoredGlobalScope.cs | 28 ------- .../Impl/Documents/RunningDocumentTable.cs | 4 +- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 2 +- .../Modules/CompiledBuiltinPythonModule.cs | 4 +- .../Ast/Impl/Modules/CompiledPythonModule.cs | 4 +- .../Definitions/ModuleCreationOptions.cs | 5 -- .../Ast/Impl/Modules/DependencyProvider.cs | 48 ------------ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 19 +---- .../Ast/Impl/Modules/PythonVariableModule.cs | 5 -- .../Resolution/MainModuleResolution.cs | 39 +++++----- .../Ast/Impl/Modules/SpecializedModule.cs | 7 +- .../Ast/Impl/Modules/StubPythonModule.cs | 2 +- .../Impl/Types/Definitions/IPythonModule.cs | 5 -- .../Impl/DependencyResolution/ModuleImport.cs | 1 - src/Caching/Impl/Models/FunctionModel.cs | 6 +- src/Caching/Impl/Models/ModuleModel.cs | 26 +++++-- src/Caching/Impl/Models/SubmoduleModel.cs | 38 ++++++++++ src/Caching/Impl/ModuleDatabase.cs | 41 +++++----- src/Caching/Impl/ModuleFactory.cs | 9 ++- src/Caching/Impl/ModuleUniqueId.cs | 11 ++- src/Caching/Impl/PythonDbModule.cs | 13 ++-- src/Caching/Impl/PythonLazyType.cs | 44 ----------- src/Caching/Impl/RestoredGlobalScope.cs | 46 ++++++++---- src/Caching/Test/AnalysisCachingTestBase.cs | 13 ++-- src/Caching/Test/ReferencesTests.cs | 1 - src/Caching/Test/RestoreTests.cs | 75 +++++++++++++++++++ 31 files changed, 272 insertions(+), 302 deletions(-) delete mode 100644 src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs delete mode 100644 src/Analysis/Ast/Impl/Modules/DependencyProvider.cs create mode 100644 src/Caching/Impl/Models/SubmoduleModel.cs delete mode 100644 src/Caching/Impl/PythonLazyType.cs create mode 100644 src/Caching/Test/RestoreTests.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 2b9182661..8d4b9751d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -13,18 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Dependencies; - namespace Microsoft.Python.Analysis.Analyzer { /// /// Represents document that can be analyzed asynchronously. /// internal interface IAnalyzable { - /// - /// Returns object that can calculate dependencies of this entry. - /// - IDependencyProvider DependencyProvider { get; } - /// /// Notifies document that analysis is about to begin. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index c7cff9b03..a0584c8eb 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -316,19 +316,33 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray } var foundKeys = ImmutableArray.Empty; - foreach (var key in missingKeys) { + foreach (var missingKey in missingKeys) { lock (_syncObj) { - if (_analysisEntries.TryGetValue(key, out _)) { + if (_analysisEntries.TryGetValue(missingKey, out _)) { continue; } } - var (moduleName, _, isTypeshed) = key; + var (moduleName, _, isTypeshed) = missingKey; var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution; + var module = moduleResolution.GetOrLoadModule(moduleName); if (module != null && module.ModuleType != ModuleType.Unresolved) { + foundKeys = foundKeys.Add(missingKey); var entry = GetOrCreateAnalysisEntry(module, out _); - _dependencyResolver.TryAddValue(key, entry, entry.IsUserModule, ImmutableArray.Empty); + _dependencyResolver.TryAddValue(missingKey, entry, entry.IsUserModule, ImmutableArray.Empty); + } + + if (foundKeys.Count > 0) { + foreach (var foundKey in foundKeys) { + PythonAnalyzerEntry entry; + lock (_syncObj) { + if (!_analysisEntries.TryGetValue(foundKey, out entry)) { + continue; + } + } + _dependencyResolver.TryAddValue(foundKey, entry, entry.IsUserModule, ImmutableArray.Empty); + } } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index f549fa87e..96bb4bf5f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -252,11 +253,8 @@ private HashSet FindDependencies(IPythonModule module, Python return dependencies; } - var dependencyProvider = (module as IAnalyzable)?.DependencyProvider; - var moduleDeps = dependencyProvider?.GetDependencies(ast); - if (moduleDeps != null) { - dependencies.UnionWith(moduleDeps); - } + var dw = new DependencyWalker(_module, ast); + dependencies.UnionWith(dw.Dependencies); dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index dd1a020c5..949b952d9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -348,11 +348,6 @@ private void AnalyzeEntry(IDependencyChainNode node, Python } private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode node, IPythonModule module, PythonAst ast, int version) { - var analysis = TryRestoreCachedAnalysis(node, module); - if (analysis != null) { - return analysis; - } - var walker = new ModuleWalker(_services, module, ast, _analyzerCancellationToken); ast.Walk(walker); walker.Complete(); @@ -370,26 +365,6 @@ private bool MarkNodeWalked(IDependencyChainNode node) { return isCanceled; } - private IDocumentAnalysis TryRestoreCachedAnalysis(IDependencyChainNode node, IPythonModule module) { - var moduleType = module.ModuleType; - if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(module.Name, module.FilePath) == true) { - if (_moduleDatabaseService.TryRestoreGlobalScope(module, out var gs)) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, "Restored from database: ", module.Name); - } - var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty()); - gs.ReconstructVariables(); - MarkNodeWalked(node); - return analysis; - } else { - if (_log != null) { - _log.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name); - } - } - } - return null; - } - private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { var canHaveLibraryAnalysis = false; diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index 16c8a213e..e7390198d 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -15,18 +15,15 @@ using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Caching { internal interface IModuleDatabaseService: IModuleDatabaseCache { /// - /// Creates global scope from module persistent state. - /// Global scope is then can be used to construct module analysis. + /// Restores module from database. /// - /// Python module to restore analysis for. - /// Python module global scope. - bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs); + IPythonModule RestoreModule(string moduleName, string modulePath, ModuleType moduleType); /// /// Writes module data to the database. @@ -36,6 +33,11 @@ internal interface IModuleDatabaseService: IModuleDatabaseCache { /// /// Determines if module analysis exists in the storage. /// - bool ModuleExistsInStorage(string moduleName, string filePath); + bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType); + } + + internal static class ModuleDatabaseExtensions { + public static bool ModuleExistsInStorage(this IModuleDatabaseService dbs, IPythonModule module) + => dbs.ModuleExistsInStorage(module.Name, module.FilePath, module.ModuleType); } } diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs deleted file mode 100644 index 71bf8e130..000000000 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IRestoredGlobalScope.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using Microsoft.Python.Analysis.Values; - -namespace Microsoft.Python.Analysis.Caching { - /// - /// Represents global scope that has been restored from - /// the database but has not been fully populated yet. - /// Used to attach to analysis so variables can be - /// accessed during classes and methods restoration. - /// - internal interface IRestoredGlobalScope : IGlobalScope { - void ReconstructVariables(); - } -} diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 9b2e259a0..699be8338 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -241,10 +241,10 @@ private DocumentEntry CreateDocument(ModuleCreationOptions mco) { IDocument document; switch (mco.ModuleType) { case ModuleType.Compiled when TryAddModulePath(mco): - document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsPersistent, mco.IsTypeshed, _services); + document = new CompiledPythonModule(mco.ModuleName, ModuleType.Compiled, mco.FilePath, mco.Stub, mco.IsTypeshed, _services); break; case ModuleType.CompiledBuiltin: - document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, mco.IsPersistent, _services); + document = new CompiledBuiltinPythonModule(mco.ModuleName, mco.Stub, _services); break; case ModuleType.User: TryAddModulePath(mco); diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 4bab90501..5edd88b6a 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -37,7 +37,7 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth private IPythonType _boolType; public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services) - : base(moduleName, ModuleType.Builtins, filePath, null, false, false, services) { } // TODO: builtins stub & persistence + : base(moduleName, ModuleType.Builtins, filePath, null, false, services) { } // TODO: builtins stub & persistence public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name); diff --git a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs index bf1e3d0c5..451cdbaea 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledBuiltinPythonModule.cs @@ -23,8 +23,8 @@ namespace Microsoft.Python.Analysis.Modules { /// Represents compiled module that is built into the language. /// internal sealed class CompiledBuiltinPythonModule : CompiledPythonModule { - public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, bool isPersistent, IServiceContainer services) - : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, isPersistent, false, services) { } + public CompiledBuiltinPythonModule(string moduleName, IPythonModule stub, IServiceContainer services) + : base(moduleName, ModuleType.CompiledBuiltin, MakeFakeFilePath(moduleName, services), stub, false, services) { } protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sm) ? null : new [] { "-W", "ignore", "-B", "-E", sm, "-u8", Name }; diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs index a5ddf50db..345d29754 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs @@ -26,8 +26,8 @@ namespace Microsoft.Python.Analysis.Modules { internal class CompiledPythonModule : PythonModule { protected IStubCache StubCache => Interpreter.ModuleResolution.StubCache; - public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isPersistent, bool isTypeshed, IServiceContainer services) - : base(moduleName, filePath, moduleType, stub, isPersistent, isTypeshed, services) { } + public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, bool isTypeshed, IServiceContainer services) + : base(moduleName, filePath, moduleType, stub, isTypeshed, services) { } public override string Documentation => GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty; diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs index ff57426cc..876e8ba47 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs @@ -48,11 +48,6 @@ public sealed class ModuleCreationOptions { /// public IPythonModule Stub { get; set; } - /// - /// Indicates if module is restored from database. - /// - public bool IsPersistent { get; set; } - /// /// Defines if module belongs to Typeshed and hence resolved /// via typeshed module resolution service. diff --git a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs b/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs deleted file mode 100644 index 63cb5aeb9..000000000 --- a/src/Analysis/Ast/Impl/Modules/DependencyProvider.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Caching; -using Microsoft.Python.Analysis.Dependencies; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.Analysis.Modules { - internal sealed class DependencyProvider: IDependencyProvider { - private readonly IPythonModule _module; - private readonly IModuleDatabaseService _dbService; - - public static IDependencyProvider Empty { get; } = new EmptyDependencyProvider(); - - public DependencyProvider(IPythonModule module, IServiceContainer services) { - _dbService = services.GetService(); - _module = module; - } - - #region IDependencyProvider - public ISet GetDependencies(PythonAst ast) { - // TODO: try and handle LoadFunctionDependencyModules functionality here. - var dw = new DependencyWalker(_module, ast); - return dw.Dependencies; - } - #endregion - - private sealed class EmptyDependencyProvider: IDependencyProvider { - public ISet GetDependencies(PythonAst ast) => new HashSet(); - } - } -} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 1deda15bf..8d284822d 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -86,14 +86,13 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser SetDeclaringModule(this); } - protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isPersistent, bool isTypeshed, IServiceContainer services) : + protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, bool isTypeshed, IServiceContainer services) : this(new ModuleCreationOptions { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType, Stub = stub, - IsTypeshed = isTypeshed, - IsPersistent = isPersistent + IsTypeshed = isTypeshed }, services) { } internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer services) @@ -119,7 +118,6 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s ContentState = State.Analyzed; } - IsPersistent = creationOptions.IsPersistent; IsTypeshed = creationOptions.IsTypeshed; InitializeContent(creationOptions.Content, 0); @@ -221,11 +219,6 @@ public virtual IEnumerable GetMemberNames() { /// public IPythonModule PrimaryModule { get; private set; } - /// - /// Indicates if module is restored from database. - /// - public bool IsPersistent { get; } - /// /// Defines if module belongs to Typeshed and hence resolved /// via typeshed module resolution service. @@ -411,8 +404,6 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit #endregion #region IAnalyzable - public virtual IDependencyProvider DependencyProvider => new DependencyProvider(this, Services); - public void NotifyAnalysisBegins() { lock (_syncObj) { if (_updated) { @@ -534,11 +525,7 @@ private void InitializeContent(string content, int version) { private void SetOrLoadContent(string content) { if (ContentState < State.Loading) { try { - if (IsPersistent) { - content = string.Empty; - } else { - content = content ?? LoadContent(); - } + content = content ?? LoadContent(); _buffer.SetContent(content); ContentState = State.Loaded; } catch (IOException) { } catch (UnauthorizedAccessException) { } diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index d674c2b8c..e55633034 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -51,7 +51,6 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua public BuiltinTypeId TypeId => BuiltinTypeId.Module; public Uri Uri => Module?.Uri; public override PythonMemberType MemberType => PythonMemberType.Module; - public bool IsPersistent => Module?.IsPersistent == true; public bool IsTypeshed => Module?.IsTypeshed == true; public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null) { @@ -84,9 +83,5 @@ public PythonVariableModule(IPythonModule module): base(module) { public SourceLocation IndexToLocation(int index) => (Module as ILocationConverter)?.IndexToLocation(index) ?? default; public int LocationToIndex(SourceLocation location) => (Module as ILocationConverter)?.LocationToIndex(location) ?? default; #endregion - - #region IDependencyProvider - public IDependencyProvider DependencyProvider => (Module as IAnalyzable)?.DependencyProvider; - #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 96dc997d9..f1d11e8e6 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -62,8 +62,9 @@ protected override IPythonModule CreateModule(string name) { return null; } + IPythonModule module; if (moduleImport.ModulePath != null) { - var module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); + module = GetRdt().GetDocument(new Uri(moduleImport.ModulePath)); if (module != null) { GetRdt().LockDocument(module.Uri); return module; @@ -71,31 +72,32 @@ protected override IPythonModule CreateModule(string name) { } var dbs = GetDbService(); - moduleImport.IsPersistent = dbs != null && dbs.ModuleExistsInStorage(name, moduleImport.ModulePath); - - IPythonModule stub = null; - if (!moduleImport.IsPersistent) { - // If there is a stub, make sure it is loaded and attached - // First check stub next to the module. - if (!TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) { - // If nothing found, try Typeshed. - stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); - } + module = dbs?.RestoreModule(name, moduleImport.ModulePath, ModuleType.Specialized); + if (module != null) { + Log?.Log(TraceEventType.Verbose, "Restored from database: ", name); + return module; + } - // If stub is created and its path equals to module, return that stub as module - if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { - return stub; - } + // If there is a stub, make sure it is loaded and attached + // First check stub next to the module. + if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { + // If nothing found, try Typeshed. + stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); + } + + // If stub is created and its path equals to module, return that stub as module + if (stub != null && stub.FilePath.PathEquals(moduleImport.ModulePath)) { + return stub; } if (moduleImport.IsBuiltin) { Log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - return new CompiledBuiltinPythonModule(name, stub, moduleImport.IsPersistent, Services); + return new CompiledBuiltinPythonModule(name, stub, Services); } if (moduleImport.IsCompiled) { Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, moduleImport.IsPersistent, false, Services); + return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, false, Services); } Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); @@ -105,8 +107,7 @@ protected override IPythonModule CreateModule(string name) { ModuleName = moduleImport.FullName, ModuleType = moduleImport.IsLibrary ? ModuleType.Library : ModuleType.User, FilePath = moduleImport.ModulePath, - Stub = stub, - IsPersistent = moduleImport.IsPersistent + Stub = stub }; return GetRdt().AddModule(mco); diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs index f9f545348..41bd338b2 100644 --- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs +++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; @@ -32,15 +31,11 @@ namespace Microsoft.Python.Analysis.Modules { /// internal abstract class SpecializedModule : PythonModule { protected SpecializedModule(string name, string modulePath, IServiceContainer services) - : base(name, modulePath, ModuleType.Specialized, null, false, false, services) { } + : base(name, modulePath, ModuleType.Specialized, null, false, services) { } protected override string LoadContent() { // Exceptions are handled in the base return FileSystem.FileExists(FilePath) ? FileSystem.ReadTextWithRetry(FilePath) : string.Empty; } - - #region IAnalyzable - public override IDependencyProvider DependencyProvider => Modules.DependencyProvider.Empty; - #endregion } } diff --git a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs index 7709c36c0..d01eaadfe 100644 --- a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Modules { /// internal class StubPythonModule : CompiledPythonModule { public StubPythonModule(string moduleName, string stubPath, bool isTypeshed, IServiceContainer services) - : base(moduleName, ModuleType.Stub, stubPath, null, false, isTypeshed, services) { + : base(moduleName, ModuleType.Stub, stubPath, null, isTypeshed, services) { } protected override string LoadContent() { diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs index 918ec3a51..4ff5dc0b7 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs @@ -64,11 +64,6 @@ public interface IPythonModule : IPythonType { /// IPythonModule PrimaryModule { get; } - /// - /// Indicates if module is restored from database. - /// - bool IsPersistent { get; } - /// /// Defines if module belongs to Typeshed and hence resolved /// via typeshed module resolution service. diff --git a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs index 9d95188e3..b229ab80e 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/ModuleImport.cs @@ -27,7 +27,6 @@ public class ModuleImport : IImportChildrenSource { public bool IsCompiled { get; } public bool IsLibrary { get; } public bool IsBuiltin => IsCompiled && ModulePath == null; - public bool IsPersistent { get; set; } public ModuleImport(IImportChildrenSource childrenSource, string name, string fullName, string rootPath, string modulePath, long moduleFileSize, bool isCompiled, bool isLibrary) { _childrenSource = childrenSource; diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 2c365e6c1..3f21d8ad5 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -41,12 +41,12 @@ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlo public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { // Create inner functions and classes first since function may be returning one of them. - var all = Classes.Concat(Functions).ToArray(); + var innerTypes = Classes.Concat(Functions).ToArray(); - foreach (var model in all) { + foreach (var model in innerTypes) { _function.AddMember(Name, model.Create(mf, _function, gs), overwrite: true); } - foreach (var model in all) { + foreach (var model in innerTypes) { model.Populate(mf, _function, gs); } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index e1eae7b98..e6230ab8f 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -37,6 +37,7 @@ internal sealed class ModuleModel : MemberModel { public ClassModel[] Classes { get; set; } public TypeVarModel[] TypeVars { get; set; } public NamedTupleModel[] NamedTuples { get; set; } + //public SubmoduleModel[] SubModules { get; set; } /// /// Collection of new line information for conversion of linear spans @@ -51,6 +52,9 @@ internal sealed class ModuleModel : MemberModel { [NonSerialized] private Dictionary _modelCache; + /// + /// Constructs module persistent model from analysis. + /// public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { var uniqueId = analysis.Document.GetUniqueId(services, options); if (uniqueId == null) { @@ -63,15 +67,10 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta var classes = new Dictionary(); var typeVars = new Dictionary(); var namedTuples = new Dictionary(); + //var subModules = new Dictionary(); - // Go directly through variables which names are listed in GetMemberNames - // as well as variables that are declarations. - var exportedNames = new HashSet(analysis.Document.GetMemberNames()); - foreach (var v in analysis.GlobalScope.Variables - .Where(v => exportedNames.Contains(v.Name) || - v.Source == VariableSource.Declaration || - v.Source == VariableSource.Builtin || - v.Source == VariableSource.Generic)) { + foreach (var v in analysis.Document.GetMemberNames() + .Select(x => analysis.GlobalScope.Variables[x]).ExcludeDefault()) { if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { typeVars[v.Name] = TypeVarModel.FromGeneric(v, services); @@ -107,6 +106,12 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals continue; } break; + //case IPythonModule m: + // if (!subModules.ContainsKey(m.Name)) { + // subModules[m.Name] = new SubmoduleModel(m, services); + // continue; + // } + // break; } // Do not re-declare classes and functions as variables in the model. @@ -127,6 +132,7 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals Classes = classes.Values.ToArray(), TypeVars = typeVars.Values.ToArray(), NamedTuples = namedTuples.Values.ToArray(), + //SubModules = subModules.Values.ToArray(), NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel { EndIndex = l.EndIndex, Kind = l.Kind @@ -163,6 +169,10 @@ public override MemberModel GetModel(string name) { return _modelCache.TryGetValue(name, out var model) ? model : null; } + public IEnumerable GetMemberNames() + => TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions) + .Concat(Variables)/*.Concat(SubModules)*/.Select(m => m.Name); + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); } diff --git a/src/Caching/Impl/Models/SubmoduleModel.cs b/src/Caching/Impl/Models/SubmoduleModel.cs new file mode 100644 index 000000000..3b751adc9 --- /dev/null +++ b/src/Caching/Impl/Models/SubmoduleModel.cs @@ -0,0 +1,38 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +// ReSharper disable MemberCanBePrivate.Global + +namespace Microsoft.Python.Analysis.Caching.Models { + //[Serializable] + //internal sealed class SubmoduleModel: MemberModel { + // public string UniqueId { get; set; } + // public string FilePath { get; set; } + // public string Documentation { get; set; } + + // public SubmoduleModel(IPythonModule m, IServiceContainer services) { + // var uniqueId = m.GetUniqueId(services); + // Id = uniqueId.GetStableHash(); + // UniqueId = uniqueId; + // Name = m.Name; + // QualifiedName = m.QualifiedName; + // FilePath = m.FilePath; + // Documentation = m.Documentation; + // } + //} +} diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 788ddc38a..9d2eee349 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -16,7 +16,6 @@ using System; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using LiteDB; @@ -34,14 +33,16 @@ internal sealed class ModuleDatabase : IModuleDatabaseService { private readonly IServiceContainer _services; private readonly ILogger _log; private readonly IFileSystem _fs; + private readonly AnalysisCachingLevel? _cachingLevel; - public ModuleDatabase(IServiceContainer services) { + public ModuleDatabase(IServiceContainer services, string cacheFolder = null, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { _services = services; _log = services.GetService(); _fs = services.GetService(); + _cachingLevel = cachingLevel; var cfs = services.GetService(); - CacheFolder = Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); + CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); } public string CacheFolderBaseName => "analysis.v"; @@ -52,22 +53,18 @@ public ModuleDatabase(IServiceContainer services) { /// Creates global scope from module persistent state. /// Global scope is then can be used to construct module analysis. /// - /// Python module to restore analysis for. - /// Python module global scope. - public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs) { - gs = null; - - if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) { - return false; + public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleType moduleType) { + if (GetCachingLevel() == AnalysisCachingLevel.None) { + return null; } lock (_lock) { - if (FindModuleModelByPath(module.Name, module.FilePath, out var model)) { - gs = new RestoredGlobalScope(model, module, this, _services); + if (FindModuleModelByPath(moduleName, modulePath, moduleType, out var model)) { + return new PythonDbModule(model, modulePath, _services); } } - return gs != null; + return null; } /// @@ -79,14 +76,14 @@ public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationTok /// /// Determines if module analysis exists in the storage. /// - public bool ModuleExistsInStorage(string moduleName, string filePath) { + public bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType) { if (GetCachingLevel() == AnalysisCachingLevel.None) { return false; } for (var retries = 50; retries > 0; --retries) { try { - var dbPath = FindDatabaseFile(moduleName, filePath); + var dbPath = FindDatabaseFile(name, filePath, moduleType); return !string.IsNullOrEmpty(dbPath); } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { Thread.Sleep(10); @@ -143,8 +140,8 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c /// by name, version, current Python interpreter version and/or hash of the /// module content (typically file sizes). /// - private string FindDatabaseFile(string moduleName, string filePath) { - var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel()); + private string FindDatabaseFile(string moduleName, string filePath, ModuleType moduleType) { + var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, GetCachingLevel()); return string.IsNullOrEmpty(uniqueId) ? null : FindDatabaseFile(uniqueId); } @@ -170,11 +167,11 @@ private string FindDatabaseFile(string uniqueId) { return _fs.FileExists(dbPath) ? dbPath : null; } - public bool FindModuleModelByPath(string moduleName, string filePath, out ModuleModel model) - => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, filePath), out model); + public bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) + => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, modulePath, moduleType), out model); - public bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) - => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, uniqueId), out model); + public bool FindModuleModelById(string moduleName, string uniqueId, ModuleType moduleType, out ModuleModel model) + => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, uniqueId, moduleType), out model); private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel model) { model = null; @@ -203,6 +200,6 @@ private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel return false; } private AnalysisCachingLevel GetCachingLevel() - => _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; + => _cachingLevel ?? _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 3bdf9a264..c3b9f6c40 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -48,11 +48,11 @@ private static readonly ConcurrentDictionary _modulesCac public IPythonModule Module { get; } public Location DefaultLocation { get; } - public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs, ModuleDatabase db, IServiceContainer services) { + public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs, IServiceContainer services) { _model = model; _gs = gs; - _db = db; _services = services; + _db = services.GetService(); Module = module; DefaultLocation = new Location(Module); } @@ -146,12 +146,13 @@ private IPythonModule GetModule(QualifiedNameParts parts) { return null; } + // If module is loaded, then use it. Otherwise, create DB module but don't restore it just yet. var module = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); if (module == null && parts.ModuleId != null) { if (!_modulesCache.TryGetValue(parts.ModuleId, out var m)) { - if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, out var model)) { + if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, ModuleType.Specialized, out var model)) { + // Create db module, but do not reconstruct the analysis just yet. _modulesCache[parts.ModuleId] = m = new PythonDbModule(model, model.FilePath, _services); - m.Construct(model); module = m; } } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index bc1da403f..c0a55f97b 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -35,10 +35,6 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType if(cachingLevel == AnalysisCachingLevel.None) { return null; } - if (moduleType == ModuleType.User) { - // Only for tests. - return $"{moduleName}"; - } var interpreter = services.GetService(); var fs = services.GetService(); @@ -86,6 +82,10 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType } var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); + if (parent == null) { + return moduleName; + } + var hash = HashModuleFileSizes(parent); // If all else fails, hash modules file sizes. return $"{moduleName}.{(ulong)hash}"; @@ -97,6 +97,9 @@ private static long HashModuleFileSizes(IImportChildrenSource source) { foreach (var name in names) { if (source.TryGetChildImport(name, out var child)) { if (child is ModuleImport moduleImport) { + if (moduleImport.ModuleFileSize == 0) { + continue; // Typically test case, memory-only module. + } hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize); } diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index bd3182e31..064d61a9d 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -15,8 +15,10 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; @@ -31,18 +33,17 @@ public PythonDbModule(ModuleModel model, string filePath, IServiceContainer serv Documentation = model.Documentation; _newLines = model.NewLines.Select(nl => new NewLineLocation(nl.EndIndex, nl.Kind)).ToArray(); _fileSize = model.FileSize; - } - - public void Construct(ModuleModel model) { - var gs = new RestoredGlobalScope(model, this, Services.GetService(), Services); - GlobalScope = gs; - gs.ReconstructVariables(); + GlobalScope = new RestoredGlobalScope(model, this, Services); } protected override string LoadContent() => string.Empty; public override string Documentation { get; } public override IEnumerable GetMemberNames() => GlobalScope.Variables.Names; + public override IMember GetMember(string name) { + ((RestoredGlobalScope)GlobalScope).ReconstructVariable(name); + return base.GetMember(name); + } #region ILocationConverter public override SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(_newLines, index); diff --git a/src/Caching/Impl/PythonLazyType.cs b/src/Caching/Impl/PythonLazyType.cs deleted file mode 100644 index d6e0deded..000000000 --- a/src/Caching/Impl/PythonLazyType.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using Microsoft.Python.Analysis.Types; - -namespace Microsoft.Python.Analysis.Caching { - internal sealed class PythonLazyType: PythonTypeWrapper { - private readonly PythonDbModule _dbModule; - - public PythonLazyType(string name, string qualifiedTypeName, string documentation, PythonDbModule dbModule, BuiltinTypeId typeid, ModuleDatabase db) - : base(name, documentation, declaringModule, ) { - _dbModule = dbModule; - TypeId = typeid; - - } - - public override BuiltinTypeId TypeId { get; } - - public override IEnumerable GetMemberNames() { - return base.GetMemberNames(); - } - - public override IMember GetMember(string name) { - return base.GetMember(name); - } - - private void EnsureInnerType() { - _dbModule - } - } -} diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index b57292046..c55b2e73c 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -23,24 +23,30 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching { - internal sealed class RestoredGlobalScope : IRestoredGlobalScope { + internal sealed class RestoredGlobalScope : IGlobalScope { private readonly VariableCollection _scopeVariables = new VariableCollection(); private ModuleModel _model; // Non-readonly b/c of DEBUG conditional. private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional. + private bool _typeVarsCreated; - public RestoredGlobalScope(ModuleModel model, IPythonModule module, ModuleDatabase db, IServiceContainer services) { + public RestoredGlobalScope(ModuleModel model, IPythonModule module, IServiceContainer services) { _model = model ?? throw new ArgumentNullException(nameof(model)); Module = module ?? throw new ArgumentNullException(nameof(module)); Name = model.Name; - _factory = new ModuleFactory(_model, Module, this, db, services); - DeclareVariables(db, services); + _factory = new ModuleFactory(_model, Module, this, services); + DeclareVariables(services); } - public void ReconstructVariables() { - var models = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); - foreach (var m in models.Concat(_model.Variables)) { - m.Populate(_factory, null, this); - } + public void ReconstructVariable(string name) { + EnsureTypeVars(); + var model = _model.NamedTuples.FirstOrDefault(x => x.Name == name) + ?? _model.Classes.Cast().FirstOrDefault(x => x.Name == name) + ?? _model.Functions.Cast().FirstOrDefault(x => x.Name == name) + //?? _model.SubModules.FirstOrDefault(x => x.Name == name) + ?? _model.Variables.FirstOrDefault(x => x.Name == name); + + model?.Populate(_factory, null, this); + // TODO: re-declare __doc__, __name__, etc. #if !DEBUG _model = null; @@ -48,18 +54,32 @@ public void ReconstructVariables() { #endif } - private void DeclareVariables(ModuleDatabase db, IServiceContainer services) { + private void EnsureTypeVars() { + if (!_typeVarsCreated) { + foreach (var m in _model.TypeVars) { + m.Populate(_factory, null, this); + } + _typeVarsCreated = true; + } + } + + private void DeclareVariables(IServiceContainer services) { // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus first create members so we can find then, then populate them with content. - var mf = new ModuleFactory(_model, Module, this, db, services); + var mf = new ModuleFactory(_model, Module, this, services); // Generics first - var typeVars = _model.TypeVars.Concat(_model.NamedTuples).Concat(_model.Classes).Concat(_model.Functions); - foreach (var m in typeVars) { + foreach (var m in _model.TypeVars) { _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Generic, mf.DefaultLocation); } + var members = _model.NamedTuples + .Concat(_model.Classes).Concat(_model.Functions); //.Concat(_model.SubModules); + foreach (var m in members) { + _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Declaration, mf.DefaultLocation); + } + // Declare variables in the order of appearance since later variables // may use types declared in the preceding ones. foreach (var vm in _model.Variables.OrderBy(m => m.IndexSpan.Start)) { diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index f81cb5657..e5eb627d2 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -27,7 +27,7 @@ using TestUtilities; namespace Microsoft.Python.Analysis.Caching.Tests { - public abstract class AnalysisCachingTestBase: AnalysisTestBase { + public abstract class AnalysisCachingTestBase : AnalysisTestBase { protected AnalysisCachingTestBase() { ModuleFactory.EnableMissingMemberAssertions = true; } @@ -51,16 +51,13 @@ protected string BaselineFilesFolder { } } - protected string GetBaselineFileName(string testName, string suffix = null) - => Path.ChangeExtension(suffix == null + protected string GetBaselineFileName(string testName, string suffix = null) + => Path.ChangeExtension(suffix == null ? Path.Combine(BaselineFilesFolder, testName) : Path.Combine(BaselineFilesFolder, testName + suffix), "json"); - internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { - var dbModule = new PythonDbModule(model, modulePath, Services); - dbModule.Construct(model); - return dbModule; - } + internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) + => new PythonDbModule(model, modulePath, Services); internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) { //var json = ToJson(model); diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs index 76b9ac803..745fca71a 100644 --- a/src/Caching/Test/ReferencesTests.cs +++ b/src/Caching/Test/ReferencesTests.cs @@ -66,7 +66,6 @@ def methodB2(self): Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { - dbModule.Construct(model); var sum = dbModule.GetMember("sum") as IPythonFunctionType; sum.Should().NotBeNull(); diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs new file mode 100644 index 000000000..4c930c542 --- /dev/null +++ b/src/Caching/Test/RestoreTests.cs @@ -0,0 +1,75 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Caching.Tests { + [TestClass] + public class RestoreTests : AnalysisCachingTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task ReturnType() { + const string code = @" +from module2 import func2 +x = func2() +"; + const string mod2Code = @" +class C2: + def M1C2(self): + return 0 + +def func2() -> C2: ... +"; + await TestData.CreateTestSpecificFileAsync("module2.py", mod2Code); + var analysis = await GetAnalysisAsync(code); + + var func2 = analysis.Should().HaveVariable("func2").Which; + var analysis2 = ((IPythonFunctionType)func2.Value).DeclaringModule.Analysis; + + var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); + Services.AddService(dbs); + await dbs.StoreModuleAnalysisAsync(analysis2, CancellationToken.None); + + await Services.GetService().ResetAnalyzer(); + var doc = Services.GetService().GetDocument(analysis.Document.Uri); + analysis = await doc.GetAnalysisAsync(Timeout.Infinite); + + analysis.Should().HaveFunction("func2") + .Which.Should().HaveSingleOverload() + .Which.Should().HaveReturnType("C2") + .Which.Should().HaveMember("M1C2") + .Which.Should().HaveSingleOverload() + .Which.Should().HaveReturnType(BuiltinTypeId.Int); + } + } +} From 96129901f772309ff9dc0b6bd7b151bd4c05d7e2 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 1 Nov 2019 20:01:02 -0700 Subject: [PATCH 007/102] Test pass --- .../Ast/Impl/Types/PythonFunctionOverload.cs | 2 +- src/Caching/Impl/Models/ClassModel.cs | 41 +++++++++++-------- src/Caching/Impl/Models/FunctionModel.cs | 34 +++++++++------ src/Caching/Impl/Models/MemberModel.cs | 27 +++++++++--- src/Caching/Impl/Models/ModuleModel.cs | 4 +- src/Caching/Impl/Models/NamedTupleModel.cs | 14 +++---- src/Caching/Impl/Models/PropertyModel.cs | 15 ++++--- src/Caching/Impl/Models/TypeVarModel.cs | 10 ++--- src/Caching/Impl/Models/VariableModel.cs | 8 ++-- src/Caching/Impl/ModuleFactory.cs | 7 +++- src/Caching/Impl/RestoredGlobalScope.cs | 10 ++--- 11 files changed, 104 insertions(+), 68 deletions(-) diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index e394de6a2..c5a97d46c 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -124,7 +124,7 @@ public string GetReturnDocumentation(IPythonType self = null) { public IMember Call(IArgumentSet args, IPythonType self) { if (!_fromAnnotation) { // First try supplied specialization callback. - var rt = _returnValueProvider?.Invoke(args.Eval.Module, this, args, default); + var rt = _returnValueProvider?.Invoke(args.Eval?.Module, this, args, default); if (!rt.IsUnknown()) { return rt; } diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index 16582e247..a92b75349 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -53,6 +53,9 @@ internal sealed class ClassModel : MemberModel { public ClassModel() { } // For de-serializer from JSON + /// + /// Constructs class model for persistence off the class in-memory type. + /// public ClassModel(IPythonClassType cls, IServiceContainer services) { var methods = new List(); var properties = new List(); @@ -130,15 +133,24 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { .ToArray(); } - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { - if(_cls != null) { - return _cls; + /// + /// Restores class from its model for declarations. The class may not be fully constructed + /// yet: method overloads and return types of methods may be missing. + /// + protected override IMember DeclareMember(IPythonType declaringType) { + if (_cls == null) { + _cls = new PythonClassType(Name, new Location(_mf.Module, IndexSpan.ToSpan())); + _cls.SetDocumentation(Documentation); } - _cls = new PythonClassType(Name, new Location(mf.Module, IndexSpan.ToSpan())); - var bases = CreateBases(mf, gs); + return _cls; + } + /// + /// Populates class with members. + /// + protected override void FinalizeMember() { + var bases = CreateBases(_mf, _gs); _cls.SetBases(bases); - _cls.SetDocumentation(Documentation); if (GenericParameterValues.Length > 0) { _cls.StoreGenericParameters( @@ -146,29 +158,24 @@ public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlo _cls.GenericParameters.Keys.ToArray(), GenericParameterValues.ToDictionary( k => _cls.GenericParameters.Keys.First(x => x == k.Name), - v => mf.ConstructType(v.Type) + v => _mf.ConstructType(v.Type) ) ); } - var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); + var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields).ToArray(); foreach (var m in all) { - _cls.AddMember(m.Name, m.Create(mf, _cls, gs), false); + _cls.AddMember(m.Name, m.Declare(_mf, _cls, _gs), false); } - return _cls; - } - - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { - var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields); foreach (var m in all) { - m.Populate(mf, _cls, gs); + m.Finalize(); } } private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { var ntBases = NamedTupleBases.Select(ntb => { - var n = ntb.Create(mf, _cls, gs); - ntb.Populate(mf, _cls, gs); + var n = ntb.Declare(mf, _cls, gs); + ntb.Finalize(); return n; }).OfType().ToArray(); diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 3f21d8ad5..b82a6eed7 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -36,26 +36,34 @@ public FunctionModel(IPythonFunctionType func, IServiceContainer services) : bas Overloads = func.Overloads.Select(s => FromOverload(s, services)).ToArray(); } - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) - => _function ?? (_function = new PythonFunctionType(Name, new Location(mf.Module, IndexSpan.ToSpan()), declaringType, Documentation)); + protected override IMember DeclareMember(IPythonType declaringType) { + Debug.Assert(_function == null); + _function = new PythonFunctionType(Name, new Location(_mf.Module, IndexSpan.ToSpan()), declaringType, Documentation); + // TODO: restore signature string so hover does not need to restore function + // parameters and return type just to look at them. + for (var i = 0; i < Overloads.Length; i++) { + var o = new PythonFunctionOverload(_function, new Location(_mf.Module, IndexSpan.ToSpan())); + o.SetDocumentation(Documentation); + _function.AddOverload(o); + } + return _function; + } - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { - // Create inner functions and classes first since function may be returning one of them. + protected override void FinalizeMember() { + // DeclareMember inner functions and classes first since function may be returning one of them. var innerTypes = Classes.Concat(Functions).ToArray(); - foreach (var model in innerTypes) { - _function.AddMember(Name, model.Create(mf, _function, gs), overwrite: true); + _function.AddMember(Name, model.Declare(_mf, _function, _gs), overwrite: true); } foreach (var model in innerTypes) { - model.Populate(mf, _function, gs); + model.Finalize(); } - foreach (var om in Overloads) { - var o = new PythonFunctionOverload(_function, new Location(mf.Module, IndexSpan.ToSpan())); - o.SetDocumentation(Documentation); - o.SetReturnValue(mf.ConstructMember(om.ReturnType), true); - o.SetParameters(om.Parameters.Select(p => ConstructParameter(mf, p)).ToArray()); - _function.AddOverload(o); + for (var i = 0; i < Overloads.Length; i++) { + var om = Overloads[i]; + var o = (PythonFunctionOverload)_function.Overloads[i]; + o.SetReturnValue(_mf.ConstructMember(om.ReturnType), true); + o.SetParameters(om.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); } } diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index 94779f915..bc04dc9b3 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -24,6 +24,10 @@ namespace Microsoft.Python.Analysis.Caching.Models { [Serializable] internal abstract class MemberModel { + [NonSerialized] protected ModuleFactory _mf; + [NonSerialized] protected IGlobalScope _gs; + [NonSerialized] private bool _finalizing; + /// /// Member unique id in the database. /// @@ -50,16 +54,29 @@ internal abstract class MemberModel { public IndexSpanModel IndexSpan { get; set; } /// - /// Create member for declaration but does not construct its parts just yet. + /// Creates member for declaration but does not construct its parts just yet. /// Used as a first pass in two-pass handling of forward declarations. /// - public abstract IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); + public IMember Declare(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + _mf = mf; + _gs = gs; + return DeclareMember(declaringType); + } /// - /// Populate member with content, such as create class methods, etc. + /// Populates member with content, such as create class methods, etc. /// - public abstract void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs); - + public void Finalize() { + if (!_finalizing) { + _finalizing = true; + FinalizeMember(); + _finalizing = false; + } + } + + protected abstract IMember DeclareMember(IPythonType declaringType); + protected abstract void FinalizeMember(); + public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name); protected virtual IEnumerable GetMemberModels() => Enumerable.Empty(); } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index e6230ab8f..b270dfb62 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -173,7 +173,7 @@ public IEnumerable GetMemberNames() => TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions) .Concat(Variables)/*.Concat(SubModules)*/.Select(m => m.Name); - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) => throw new NotImplementedException(); + protected override IMember DeclareMember(IPythonType declaringType) => throw new NotImplementedException(); + protected override void FinalizeMember() => throw new NotImplementedException(); } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index 88591b86f..93dc9e49c 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -36,23 +36,21 @@ public NamedTupleModel() { } // For de-serializer from JSON public NamedTupleModel(ITypingNamedTupleType nt, IServiceContainer services) { Id = nt.Name.GetStableHash(); Name = nt.Name; - DeclaringModuleId = nt.DeclaringModule.GetUniqueId(services, AnalysisCachingLevel.Library); + DeclaringModuleId = nt.DeclaringModule.GetUniqueId(services); QualifiedName = nt.QualifiedName; IndexSpan = nt.Location.IndexSpan.ToModel(); ItemNames = nt.ItemNames.ToArray(); ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray(); } - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { - if (_namedTuple != null) { - return _namedTuple; + protected override IMember DeclareMember(IPythonType declaringType) { + if (_namedTuple == null) { + var itemTypes = ItemTypes.Select(_mf.ConstructType).ToArray(); + _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, _mf.Module, IndexSpan.ToSpan()); } - - var itemTypes = ItemTypes.Select(mf.ConstructType).ToArray(); - _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, mf.Module, IndexSpan.ToSpan()); return _namedTuple; } - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } + protected override void FinalizeMember() { } } } diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index 676dd6a7d..1bc37a6d6 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -33,15 +34,17 @@ public PropertyModel(IPythonPropertyType prop, IServiceContainer services) : bas ReturnType = prop.ReturnType.GetPersistentQualifiedName(services); } - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) - => _property ?? (_property = new PythonPropertyType(Name, new Location(mf.Module, IndexSpan.ToSpan()), Documentation, declaringType, (Attributes & FunctionAttributes.Abstract) != 0)); - - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + protected override IMember DeclareMember(IPythonType declaringType) { + Debug.Assert(_property == null); + _property = new PythonPropertyType(Name, new Location(_mf.Module, IndexSpan.ToSpan()), Documentation, declaringType, (Attributes & FunctionAttributes.Abstract) != 0); _property.SetDocumentation(Documentation); + return _property; + } - var o = new PythonFunctionOverload(_property, mf.DefaultLocation); + protected override void FinalizeMember() { + var o = new PythonFunctionOverload(_property, _mf.DefaultLocation); o.SetDocumentation(Documentation); - o.SetReturnValue(mf.ConstructMember(ReturnType), true); + o.SetReturnValue(_mf.ConstructMember(ReturnType), true); _property.AddOverload(o); } } diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index fd38dee3d..0857508f2 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -45,11 +45,11 @@ public static TypeVarModel FromGeneric(IVariable v, IServiceContainer services) }; } - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) - => new GenericTypeParameter(Name, mf.Module, - Constraints.Select(mf.ConstructType).ToArray(), - mf.ConstructType(Bound), Covariant, Contravariant, default); + protected override IMember DeclareMember(IPythonType declaringType) + => new GenericTypeParameter(Name, _mf.Module, + Constraints.Select(_mf.ConstructType).ToArray(), + _mf.ConstructType(Bound), Covariant, Contravariant, default); - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } + protected override void FinalizeMember() { } } } diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index 0f37ad7b6..791bb5895 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -49,11 +49,11 @@ internal sealed class VariableModel : MemberModel { Value = t.GetPersistentQualifiedName(services) }; - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { - var m = mf.ConstructMember(Value) ?? mf.Module.Interpreter.UnknownType; - return new Variable(Name, m, VariableSource.Declaration, new Location(mf.Module, IndexSpan?.ToSpan() ?? default)); + protected override IMember DeclareMember(IPythonType declaringType) { + var m = _mf.ConstructMember(Value) ?? _mf.Module.Interpreter.UnknownType; + return new Variable(Name, m, VariableSource.Declaration, new Location(_mf.Module, IndexSpan?.ToSpan() ?? default)); } - public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } + protected override void FinalizeMember() { } } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index c3b9f6c40..82fcfdb0e 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -114,8 +114,11 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { return null; } - m = nextModel.Create(this, declaringType, _gs); + m = nextModel.Declare(this, declaringType, _gs); Debug.Assert(m != null); + if (m != null) { + nextModel.Finalize(); + } if (m is IGenericType gt && typeArgs.Count > 0) { m = gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)); @@ -151,7 +154,7 @@ private IPythonModule GetModule(QualifiedNameParts parts) { if (module == null && parts.ModuleId != null) { if (!_modulesCache.TryGetValue(parts.ModuleId, out var m)) { if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, ModuleType.Specialized, out var model)) { - // Create db module, but do not reconstruct the analysis just yet. + // DeclareMember db module, but do not reconstruct the analysis just yet. _modulesCache[parts.ModuleId] = m = new PythonDbModule(model, model.FilePath, _services); module = m; } diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index c55b2e73c..f427fa423 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -45,7 +45,7 @@ public void ReconstructVariable(string name) { //?? _model.SubModules.FirstOrDefault(x => x.Name == name) ?? _model.Variables.FirstOrDefault(x => x.Name == name); - model?.Populate(_factory, null, this); + model?.Finalize(); // TODO: re-declare __doc__, __name__, etc. #if !DEBUG @@ -57,7 +57,7 @@ public void ReconstructVariable(string name) { private void EnsureTypeVars() { if (!_typeVarsCreated) { foreach (var m in _model.TypeVars) { - m.Populate(_factory, null, this); + m.Finalize(); } _typeVarsCreated = true; } @@ -71,19 +71,19 @@ private void DeclareVariables(IServiceContainer services) { // Generics first foreach (var m in _model.TypeVars) { - _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Generic, mf.DefaultLocation); + _scopeVariables.DeclareVariable(m.Name, m.Declare(mf, null, this), VariableSource.Generic, mf.DefaultLocation); } var members = _model.NamedTuples .Concat(_model.Classes).Concat(_model.Functions); //.Concat(_model.SubModules); foreach (var m in members) { - _scopeVariables.DeclareVariable(m.Name, m.Create(mf, null, this), VariableSource.Declaration, mf.DefaultLocation); + _scopeVariables.DeclareVariable(m.Name, m.Declare(mf, null, this), VariableSource.Declaration, mf.DefaultLocation); } // Declare variables in the order of appearance since later variables // may use types declared in the preceding ones. foreach (var vm in _model.Variables.OrderBy(m => m.IndexSpan.Start)) { - var v = (IVariable)vm.Create(mf, null, this); + var v = (IVariable)vm.Declare(mf, null, this); _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); } } From 2fa422b4f8885d8100e90e052c1bc0934c764326 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sun, 3 Nov 2019 17:21:20 -0700 Subject: [PATCH 008/102] Partial proxies --- .../Ast/Impl/Types/PythonTypeWrapper.cs | 9 +- src/Caching/Impl/Lazy/PythonLazyClassType.cs | 159 ++++++++++++++++++ .../Impl/Lazy/PythonLazyFunctionOverload.cs | 45 +++++ .../Impl/Lazy/PythonLazyFunctionType.cs | 79 +++++++++ .../Impl/Lazy/PythonLazyPropertyType.cs | 44 +++++ src/Caching/Impl/Lazy/PythonLazyType.cs | 39 +++++ src/Caching/Impl/Models/ClassModel.cs | 38 +---- src/Caching/Impl/Models/FunctionModel.cs | 37 ---- src/Caching/Impl/Models/MemberModel.cs | 30 ---- src/Caching/Impl/Models/ModuleModel.cs | 2 +- src/Caching/Impl/Models/NamedTupleModel.cs | 2 +- src/Caching/Impl/Models/PropertyModel.cs | 2 +- src/Caching/Impl/Models/TypeVarModel.cs | 2 +- src/Caching/Impl/Models/VariableModel.cs | 2 +- src/Caching/Impl/ModuleDatabase.cs | 12 +- src/Caching/Impl/ModuleFactory.cs | 14 +- src/Caching/Impl/RestoredGlobalScope.cs | 12 +- 17 files changed, 401 insertions(+), 127 deletions(-) create mode 100644 src/Caching/Impl/Lazy/PythonLazyClassType.cs create mode 100644 src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs create mode 100644 src/Caching/Impl/Lazy/PythonLazyFunctionType.cs create mode 100644 src/Caching/Impl/Lazy/PythonLazyPropertyType.cs create mode 100644 src/Caching/Impl/Lazy/PythonLazyType.cs diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index 3c38194b1..95c0a4abc 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -34,7 +34,7 @@ protected IPythonType InnerType /// Creates delegate type wrapper over an existing type. /// Use dedicated constructor for wrapping builtin types. /// - public PythonTypeWrapper(IPythonType type) : this(type, type.DeclaringModule) { } + protected PythonTypeWrapper(IPythonType type) : this(type, type.DeclaringModule) { } public PythonTypeWrapper(string typeName, string documentation, IPythonModule declaringModule, IPythonType baseType) : this(baseType, declaringModule) { _typeName = typeName; @@ -45,7 +45,7 @@ public PythonTypeWrapper(string typeName, string documentation, IPythonModule de /// Creates delegate type wrapper over an existing type. /// Use dedicated constructor for wrapping builtin types. /// - public PythonTypeWrapper(IPythonType type, IPythonModule declaringModule) { + protected PythonTypeWrapper(IPythonType type, IPythonModule declaringModule) { _innerType = type ?? throw new ArgumentNullException(nameof(type)); DeclaringModule = declaringModule; } @@ -55,11 +55,14 @@ public PythonTypeWrapper(IPythonType type, IPythonModule declaringModule) { /// wrap builtins since it can be done when builtins module is not loaded /// yet - such as when builtins module itself is being imported or specialized. /// - public PythonTypeWrapper(BuiltinTypeId builtinTypeId, IPythonModule declaringModule) { + protected PythonTypeWrapper(BuiltinTypeId builtinTypeId, IPythonModule declaringModule) { DeclaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule)); _builtinTypeId = builtinTypeId; } + protected PythonTypeWrapper() { } + protected void SetInnerType(IPythonType innerType) => _innerType = innerType; + #region IPythonType public virtual string Name => _typeName ?? InnerType.Name; public virtual string QualifiedName => _typeName != null ? $"{DeclaringModule.Name}:{_typeName}" : InnerType.QualifiedName; diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs new file mode 100644 index 000000000..57e967e64 --- /dev/null +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -0,0 +1,159 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal sealed class PythonLazyClassType : PythonLazyType, IPythonClassType { + private readonly PythonClassType _cls; + + public PythonLazyClassType(ClassModel model, ModuleFactory mf, IPythonType declaringType) + : base(model, mf, null, declaringType) { + _model = model ?? throw new ArgumentNullException(nameof(model)); + _cls = new PythonClassType(Name, new Location(mf.Module, model.IndexSpan.ToSpan())); + _cls.SetDocumentation(Documentation); + SetInnerType(_cls); + } + + #region IPythonType + public override IMember GetMember(string name) { + EnsureContent(); + return base.GetMember(name); + } + + public override IEnumerable GetMemberNames() { + EnsureContent(); + return base.GetMemberNames(); + } + #endregion + + #region IPythonClassType + public IPythonType CreateSpecificType(IArgumentSet typeArguments) { + EnsureContent(); + return _cls.CreateSpecificType(typeArguments); + } + + public IPythonType DeclaringType => _cls.DeclaringType; + public IReadOnlyList Parameters { + get { + EnsureContent(); + return _cls.Parameters; + } + } + + public bool IsGeneric { + get { + EnsureContent(); + return _cls.IsGeneric; + } + } + + public ClassDefinition ClassDefinition => null; + + public IReadOnlyList Mro { + get { + EnsureContent(); + return _cls.Mro; + } + } + + public IReadOnlyList Bases { + get { + EnsureContent(); + return _cls.Bases; + } + } + + public IReadOnlyDictionary GenericParameters { + get { + EnsureContent(); + return _cls.GenericParameters; + } + } + #endregion + + protected override void EnsureContent() { + if (_model == null) { + return; + } + + var bases = CreateBases(_mf, _gs); + _cls.SetBases(bases); + + if (_model.GenericParameterValues.Length > 0) { + _cls.StoreGenericParameters( + _cls, + _cls.GenericParameters.Keys.ToArray(), + _model.GenericParameterValues.ToDictionary( + k => _cls.GenericParameters.Keys.First(x => x == k.Name), + v => _mf.ConstructType(v.Type) + ) + ); + } + + var all = _model.Classes.Concat(_model.Properties).Concat(_model.Methods).Concat(_model.Fields).ToArray(); + foreach (var m in all) { + _cls.AddMember(m.Name, m.CreateDeclaration(_mf, _cls, _gs), false); + } + foreach (var m in all) { + m.CreateContent(); + } + + ReleaseModel(); + } + + private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { + var ntBases = _model.NamedTupleBases.Select(ntb => { + var n = ntb.CreateDeclaration(mf, _cls, gs); + ntb.CreateContent(); + return n; + }).OfType().ToArray(); + + var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); + var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); + var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); + + if (_model.GenericBaseParameters.Length > 0) { + // Generic class. Need to reconstruct generic base so code can then + // create specific types off the generic class. + var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); + if (genericBase != null) { + var typeVars = _model.GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); + //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); + if (typeVars.Length > 0) { + var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); + if (genericWithParameters != null) { + bases = bases.Except(Enumerable.Repeat(genericBase, 1)).Concat(Enumerable.Repeat(genericWithParameters, 1)).ToArray(); + } + } + } else { + Debug.Fail("Generic class does not have generic base."); + } + } + return bases; + } + + } +} diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs new file mode 100644 index 000000000..0e54cc73c --- /dev/null +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs @@ -0,0 +1,45 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal sealed class PythonLazyFunctionOverload: IPythonFunctionOverload { + private OverloadModel _model; + + public PythonLazyFunctionOverload(OverloadModel model) { + _model = model; + } + + public FunctionDefinition FunctionDefinition { get; } + public IPythonClassMember ClassMember { get; } + public string Name { get; } + public string Documentation { get; } + public IReadOnlyList Parameters { get; } + public IMember Call(IArgumentSet args, IPythonType self) { + throw new NotImplementedException(); + } + + public string GetReturnDocumentation(IPythonType self = null) { + throw new NotImplementedException(); + } + + public IMember StaticReturnValue { get; } + } +} diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs new file mode 100644 index 000000000..49434c205 --- /dev/null +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -0,0 +1,79 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal sealed class PythonLazyFunctionType : PythonLazyType, IPythonFunctionType { + private readonly PythonFunctionType _function; + + public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) + : base(model, mf, gs, declaringType) { + var location = new Location(mf.Module, model.IndexSpan.ToSpan()); + _function = new PythonFunctionType(_model.Name, location, declaringType, Documentation); + + // TODO: restore signature string so hover does not need to restore function + // parameters and return type just to look at them. + for (var i = 0; i < model.Overloads.Length; i++) { + var o = new PythonFunctionOverload(_function, location); + o.SetDocumentation(Documentation); + _function.AddOverload(o); + } + SetInnerType(_function); + } + + #region IPythonFunctionType + public IPythonType DeclaringType => _function.DeclaringType; + public FunctionDefinition FunctionDefinition => null; + public bool IsClassMethod => _function.IsClassMethod; + public bool IsStatic => _function.IsStatic; + public bool IsOverload => _function.IsStatic; + public bool IsStub => _function.IsStatic; + public bool IsUnbound => _function.IsStatic; + public IReadOnlyList Overloads => _function.Overloads; + #endregion + + protected override void EnsureContent() { + if (_model == null) { + return; + } + + // DeclareMember inner functions and classes first since function may be returning one of them. + var innerTypes = _model.Classes.Concat(_model.Functions).ToArray(); + foreach (var model in innerTypes) { + _function.AddMember(Name, model.CreateDeclaration(_mf, _function, _gs), overwrite: true); + } + foreach (var model in innerTypes) { + model.CreateContent(); + } + + for (var i = 0; i < _model.Overloads.Length; i++) { + var om = _model.Overloads[i]; + var o = (PythonFunctionOverload)_function.Overloads[i]; + o.SetReturnValue(_mf.ConstructMember(om.ReturnType), true); + o.SetParameters(om.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); + } + + ReleaseModel(); + } + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) + => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); + } +} diff --git a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs new file mode 100644 index 000000000..da6e011bc --- /dev/null +++ b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs @@ -0,0 +1,44 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal sealed class PythonLazyPropertyType: PythonTypeWrapper, IPythonPropertyType { + private readonly PythonPropertyType _innerProperty; + private FunctionModel _model; + + public PythonLazyPropertyType(PythonPropertyType innerProperty) { + _innerProperty = innerProperty; + } + + public IPythonType DeclaringType => _innerProperty.DeclaringType; + public FunctionDefinition FunctionDefinition => null; + public string Description => _innerProperty.Description; + public bool IsReadOnly => _innerProperty.IsReadOnly; + public IMember ReturnType { + get { + EnsureContent(); + return _innerProperty.ReturnType; + } + } + private void EnsureContent() { + _model?.CreateContent(); + _model = null; + } + } +} diff --git a/src/Caching/Impl/Lazy/PythonLazyType.cs b/src/Caching/Impl/Lazy/PythonLazyType.cs new file mode 100644 index 000000000..dbe58199c --- /dev/null +++ b/src/Caching/Impl/Lazy/PythonLazyType.cs @@ -0,0 +1,39 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal abstract class PythonLazyType : PythonTypeWrapper where TModel : class { + protected IGlobalScope _gs; + protected ModuleFactory _mf; + protected TModel _model; + + protected PythonLazyType(TModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) { + _model = model ?? throw new ArgumentNullException(nameof(model)); + _mf = mf ?? throw new ArgumentNullException(nameof(mf)); + _gs = gs ?? throw new ArgumentNullException(nameof(gs)); + } + + protected abstract void EnsureContent(); + protected void ReleaseModel() { + _mf = null; + _gs = null; + _model = null; + } + } +} diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index a92b75349..97f1b219e 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -135,7 +135,7 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { /// /// Restores class from its model for declarations. The class may not be fully constructed - /// yet: method overloads and return types of methods may be missing. + /// yet: method overloads and return types of methods may be missing. /// protected override IMember DeclareMember(IPythonType declaringType) { if (_cls == null) { @@ -148,7 +148,7 @@ protected override IMember DeclareMember(IPythonType declaringType) { /// /// Populates class with members. /// - protected override void FinalizeMember() { + protected override void PopulateMember() { var bases = CreateBases(_mf, _gs); _cls.SetBases(bases); @@ -165,43 +165,13 @@ protected override void FinalizeMember() { var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields).ToArray(); foreach (var m in all) { - _cls.AddMember(m.Name, m.Declare(_mf, _cls, _gs), false); + _cls.AddMember(m.Name, m.CreateDeclaration(_mf, _cls, _gs), false); } foreach (var m in all) { - m.Finalize(); + m.CreateContent(); } } - private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { - var ntBases = NamedTupleBases.Select(ntb => { - var n = ntb.Declare(mf, _cls, gs); - ntb.Finalize(); - return n; - }).OfType().ToArray(); - - var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); - var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); - var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); - - if (GenericBaseParameters.Length > 0) { - // Generic class. Need to reconstruct generic base so code can then - // create specific types off the generic class. - var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); - if (genericBase != null) { - var typeVars = GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); - //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); - if (typeVars.Length > 0) { - var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); - if (genericWithParameters != null) { - bases = bases.Except(Enumerable.Repeat(genericBase, 1)).Concat(Enumerable.Repeat(genericWithParameters, 1)).ToArray(); - } - } - } else { - Debug.Fail("Generic class does not have generic base."); - } - } - return bases; - } protected override IEnumerable GetMemberModels() => Classes.Concat(Methods).Concat(Properties).Concat(Fields); diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index b82a6eed7..682f011b7 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -17,7 +17,6 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global @@ -30,46 +29,10 @@ internal sealed class FunctionModel : CallableModel { public OverloadModel[] Overloads { get; set; } public FunctionModel() { } // For de-serializer from JSON - [NonSerialized] private PythonFunctionType _function; - public FunctionModel(IPythonFunctionType func, IServiceContainer services) : base(func, services) { Overloads = func.Overloads.Select(s => FromOverload(s, services)).ToArray(); } - protected override IMember DeclareMember(IPythonType declaringType) { - Debug.Assert(_function == null); - _function = new PythonFunctionType(Name, new Location(_mf.Module, IndexSpan.ToSpan()), declaringType, Documentation); - // TODO: restore signature string so hover does not need to restore function - // parameters and return type just to look at them. - for (var i = 0; i < Overloads.Length; i++) { - var o = new PythonFunctionOverload(_function, new Location(_mf.Module, IndexSpan.ToSpan())); - o.SetDocumentation(Documentation); - _function.AddOverload(o); - } - return _function; - } - - protected override void FinalizeMember() { - // DeclareMember inner functions and classes first since function may be returning one of them. - var innerTypes = Classes.Concat(Functions).ToArray(); - foreach (var model in innerTypes) { - _function.AddMember(Name, model.Declare(_mf, _function, _gs), overwrite: true); - } - foreach (var model in innerTypes) { - model.Finalize(); - } - - for (var i = 0; i < Overloads.Length; i++) { - var om = Overloads[i]; - var o = (PythonFunctionOverload)_function.Overloads[i]; - o.SetReturnValue(_mf.ConstructMember(om.ReturnType), true); - o.SetParameters(om.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); - } - } - - private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) - => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); - private static OverloadModel FromOverload(IPythonFunctionOverload o, IServiceContainer services) => new OverloadModel { Parameters = o.Parameters.Select(p => new ParameterModel { diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index bc04dc9b3..15a5fba3f 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -16,18 +16,12 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; // ReSharper disable MemberCanBeProtected.Global // ReSharper disable UnusedAutoPropertyAccessor.Global namespace Microsoft.Python.Analysis.Caching.Models { [Serializable] internal abstract class MemberModel { - [NonSerialized] protected ModuleFactory _mf; - [NonSerialized] protected IGlobalScope _gs; - [NonSerialized] private bool _finalizing; - /// /// Member unique id in the database. /// @@ -53,30 +47,6 @@ internal abstract class MemberModel { /// public IndexSpanModel IndexSpan { get; set; } - /// - /// Creates member for declaration but does not construct its parts just yet. - /// Used as a first pass in two-pass handling of forward declarations. - /// - public IMember Declare(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { - _mf = mf; - _gs = gs; - return DeclareMember(declaringType); - } - - /// - /// Populates member with content, such as create class methods, etc. - /// - public void Finalize() { - if (!_finalizing) { - _finalizing = true; - FinalizeMember(); - _finalizing = false; - } - } - - protected abstract IMember DeclareMember(IPythonType declaringType); - protected abstract void FinalizeMember(); - public virtual MemberModel GetModel(string name) => GetMemberModels().FirstOrDefault(m => m.Name == name); protected virtual IEnumerable GetMemberModels() => Enumerable.Empty(); } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index b270dfb62..1bf35bd39 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -174,6 +174,6 @@ public IEnumerable GetMemberNames() .Concat(Variables)/*.Concat(SubModules)*/.Select(m => m.Name); protected override IMember DeclareMember(IPythonType declaringType) => throw new NotImplementedException(); - protected override void FinalizeMember() => throw new NotImplementedException(); + protected override void PopulateMember() => throw new NotImplementedException(); } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index 93dc9e49c..2dce55de3 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -51,6 +51,6 @@ protected override IMember DeclareMember(IPythonType declaringType) { return _namedTuple; } - protected override void FinalizeMember() { } + protected override void PopulateMember() { } } } diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index 1bc37a6d6..23436820e 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -41,7 +41,7 @@ protected override IMember DeclareMember(IPythonType declaringType) { return _property; } - protected override void FinalizeMember() { + protected override void PopulateMember() { var o = new PythonFunctionOverload(_property, _mf.DefaultLocation); o.SetDocumentation(Documentation); o.SetReturnValue(_mf.ConstructMember(ReturnType), true); diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index 0857508f2..4e54ffe9d 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -50,6 +50,6 @@ protected override IMember DeclareMember(IPythonType declaringType) Constraints.Select(_mf.ConstructType).ToArray(), _mf.ConstructType(Bound), Covariant, Contravariant, default); - protected override void FinalizeMember() { } + protected override void PopulateMember() { } } } diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index 791bb5895..55fd4f10d 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -54,6 +54,6 @@ protected override IMember DeclareMember(IPythonType declaringType) { return new Variable(Name, m, VariableSource.Declaration, new Location(_mf.Module, IndexSpan?.ToSpan() ?? default)); } - protected override void FinalizeMember() { } + protected override void PopulateMember() { } } } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 9d2eee349..d3af45376 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -25,6 +25,7 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.Services; namespace Microsoft.Python.Analysis.Caching { internal sealed class ModuleDatabase : IModuleDatabaseService { @@ -35,14 +36,15 @@ internal sealed class ModuleDatabase : IModuleDatabaseService { private readonly IFileSystem _fs; private readonly AnalysisCachingLevel? _cachingLevel; - public ModuleDatabase(IServiceContainer services, string cacheFolder = null, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { - _services = services; - _log = services.GetService(); - _fs = services.GetService(); + public ModuleDatabase(IServiceManager sm, string cacheFolder = null, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { + _services = sm; + _log = _services.GetService(); + _fs = _services.GetService(); _cachingLevel = cachingLevel; - var cfs = services.GetService(); + var cfs = _services.GetService(); CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); + sm.AddService(this); } public string CacheFolderBaseName => "analysis.v"; diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 82fcfdb0e..d51d67785 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -49,11 +49,11 @@ private static readonly ConcurrentDictionary _modulesCac public Location DefaultLocation { get; } public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs, IServiceContainer services) { - _model = model; - _gs = gs; - _services = services; + _model = model ?? throw new ArgumentNullException(nameof(model)); + _gs = gs ?? throw new ArgumentNullException(nameof(gs)); + _services = services ?? throw new ArgumentNullException(nameof(services)); _db = services.GetService(); - Module = module; + Module = module ?? throw new ArgumentNullException(nameof(module)); DefaultLocation = new Location(Module); } @@ -114,10 +114,10 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { return null; } - m = nextModel.Declare(this, declaringType, _gs); + m = nextModel.CreateDeclaration(this, declaringType, _gs); Debug.Assert(m != null); if (m != null) { - nextModel.Finalize(); + nextModel.CreateContent(); } if (m is IGenericType gt && typeArgs.Count > 0) { @@ -151,7 +151,7 @@ private IPythonModule GetModule(QualifiedNameParts parts) { // If module is loaded, then use it. Otherwise, create DB module but don't restore it just yet. var module = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); - if (module == null && parts.ModuleId != null) { + if (module == null && parts.ModuleId != null && _db != null) { if (!_modulesCache.TryGetValue(parts.ModuleId, out var m)) { if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, ModuleType.Specialized, out var model)) { // DeclareMember db module, but do not reconstruct the analysis just yet. diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index f427fa423..5f02fd9ab 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -45,7 +45,7 @@ public void ReconstructVariable(string name) { //?? _model.SubModules.FirstOrDefault(x => x.Name == name) ?? _model.Variables.FirstOrDefault(x => x.Name == name); - model?.Finalize(); + model?.CreateContent(); // TODO: re-declare __doc__, __name__, etc. #if !DEBUG @@ -57,7 +57,7 @@ public void ReconstructVariable(string name) { private void EnsureTypeVars() { if (!_typeVarsCreated) { foreach (var m in _model.TypeVars) { - m.Finalize(); + m.CreateContent(); } _typeVarsCreated = true; } @@ -71,19 +71,19 @@ private void DeclareVariables(IServiceContainer services) { // Generics first foreach (var m in _model.TypeVars) { - _scopeVariables.DeclareVariable(m.Name, m.Declare(mf, null, this), VariableSource.Generic, mf.DefaultLocation); + _scopeVariables.DeclareVariable(m.Name, m.CreateDeclaration(mf, null, this), VariableSource.Generic, mf.DefaultLocation); } var members = _model.NamedTuples .Concat(_model.Classes).Concat(_model.Functions); //.Concat(_model.SubModules); foreach (var m in members) { - _scopeVariables.DeclareVariable(m.Name, m.Declare(mf, null, this), VariableSource.Declaration, mf.DefaultLocation); + _scopeVariables.DeclareVariable(m.Name, m.CreateDeclaration(mf, null, this), VariableSource.Declaration, mf.DefaultLocation); } - // Declare variables in the order of appearance since later variables + // CreateDeclaration variables in the order of appearance since later variables // may use types declared in the preceding ones. foreach (var vm in _model.Variables.OrderBy(m => m.IndexSpan.Start)) { - var v = (IVariable)vm.Declare(mf, null, this); + var v = (IVariable)vm.CreateDeclaration(mf, null, this); _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); } } From 832d9813a38b6dcfe2c1f0cebbdd2d1bfc6ad175 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Nov 2019 10:48:56 -0800 Subject: [PATCH 009/102] Initial lazy types --- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 2 +- .../Types/Definitions/IPythonPropertyType.cs | 5 -- .../Ast/Impl/Types/PythonPropertyType.cs | 17 ++---- src/Caching/Impl/Lazy/MemberFactory.cs | 52 ++++++++++++++++ src/Caching/Impl/Lazy/PythonLazyClassType.cs | 44 +++++++------- .../Impl/Lazy/PythonLazyFunctionOverload.cs | 45 -------------- .../Impl/Lazy/PythonLazyFunctionType.cs | 22 ++++--- .../Impl/Lazy/PythonLazyPropertyType.cs | 36 +++++++---- src/Caching/Impl/Lazy/PythonLazyType.cs | 26 +++++--- src/Caching/Impl/Models/CallableModel.cs | 1 - src/Caching/Impl/Models/ClassModel.cs | 43 ------------- src/Caching/Impl/Models/ModuleModel.cs | 13 ++-- src/Caching/Impl/Models/NamedTupleModel.cs | 12 ---- src/Caching/Impl/Models/PropertyModel.cs | 19 +----- src/Caching/Impl/Models/TypeVarModel.cs | 9 --- src/Caching/Impl/Models/VariableModel.cs | 7 --- src/Caching/Impl/ModuleFactory.cs | 6 +- src/Caching/Impl/PythonDbModule.cs | 8 --- src/Caching/Impl/RestoredGlobalScope.cs | 60 +++++-------------- 19 files changed, 154 insertions(+), 273 deletions(-) create mode 100644 src/Caching/Impl/Lazy/MemberFactory.cs delete mode 100644 src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index a58f9f9b8..6af3c1640 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -127,7 +127,7 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) { AddOverload(fd, f, o => f.AddOverload(o)); } - private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { + private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { // Check if function exists in stubs. If so, take overload from stub // and the documentation from this actual module. if (!_table.ReplacedByStubs.Contains(fd)) { diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs index badcecb6b..a6efd3d4f 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonPropertyType.cs @@ -25,11 +25,6 @@ public interface IPythonPropertyType : IPythonClassMember { /// FunctionDefinition FunctionDefinition { get; } - /// - /// A user readable description of the property. - /// - string Description { get; } - /// /// True if the property is read-only. /// diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 12297dace..33d75e15b 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -14,13 +14,10 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { internal sealed class PythonPropertyType : PythonType, IPythonPropertyType { - private IPythonFunctionOverload _getter; - public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract) : this(fd.Name, location, fd.GetDocumentation(), declaringType, isAbstract) { declaringType.DeclaringModule.AddAstNode(this, fd); @@ -43,19 +40,13 @@ public PythonPropertyType(string name, Location location, string documentation, public bool IsReadOnly => true; public IPythonType DeclaringType { get; } - public string Description { - get { - var typeName = ReturnType?.GetPythonType()?.Name; - return typeName != null ? Resources.PropertyOfType.FormatUI(typeName) : Resources.PropertyOfUnknownType; - } - } - public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType); + => Getter.Call(args, instance?.GetPythonType() ?? DeclaringType); - public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType); + public IMember ReturnType => Getter?.Call(ArgumentSet.WithoutContext, DeclaringType); #endregion - internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload; + internal void AddOverload(PythonFunctionOverload overload) => Getter = overload; + internal PythonFunctionOverload Getter { get; private set; } } } diff --git a/src/Caching/Impl/Lazy/MemberFactory.cs b/src/Caching/Impl/Lazy/MemberFactory.cs new file mode 100644 index 000000000..f84f96678 --- /dev/null +++ b/src/Caching/Impl/Lazy/MemberFactory.cs @@ -0,0 +1,52 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal static class MemberFactory { + public static IMember CreateMember(MemberModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) { + switch (model) { + case ClassModel cm: + return new PythonLazyClassType(cm, mf, declaringType); + case FunctionModel fm: + return new PythonLazyFunctionType(fm, mf, gs, declaringType); + case PropertyModel pm: + return new PythonLazyPropertyType(pm, mf, gs, declaringType); + + case NamedTupleModel ntm: + var itemTypes = ntm.ItemTypes.Select(mf.ConstructType).ToArray(); + return new NamedTupleType(ntm.Name, ntm.ItemNames, itemTypes, mf.Module, ntm.IndexSpan.ToSpan()); + + case TypeVarModel tvm: + return new GenericTypeParameter(tvm.Name, mf.Module, + tvm.Constraints.Select(mf.ConstructType).ToArray(), + mf.ConstructType(tvm.Bound), tvm.Covariant, tvm.Contravariant, default); + + case VariableModel vm: + var m = mf.ConstructMember(vm.Value) ?? mf.Module.Interpreter.UnknownType; + return new Variable(vm.Name, m, VariableSource.Declaration, new Location(mf.Module, vm.IndexSpan?.ToSpan() ?? default)); + + } + Debug.Fail("Unsupported model type."); + return null; + } + } +} diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs index 57e967e64..2df5fb726 100644 --- a/src/Caching/Impl/Lazy/PythonLazyClassType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -31,7 +31,6 @@ internal sealed class PythonLazyClassType : PythonLazyType, IPythonC public PythonLazyClassType(ClassModel model, ModuleFactory mf, IPythonType declaringType) : base(model, mf, null, declaringType) { - _model = model ?? throw new ArgumentNullException(nameof(model)); _cls = new PythonClassType(Name, new Location(mf.Module, model.IndexSpan.ToSpan())); _cls.SetDocumentation(Documentation); SetInnerType(_cls); @@ -55,7 +54,6 @@ public IPythonType CreateSpecificType(IArgumentSet typeArguments) { return _cls.CreateSpecificType(typeArguments); } - public IPythonType DeclaringType => _cls.DeclaringType; public IReadOnlyList Parameters { get { EnsureContent(); @@ -95,52 +93,53 @@ public IReadOnlyDictionary GenericParameters { #endregion protected override void EnsureContent() { - if (_model == null) { + if (Model == null) { return; } - var bases = CreateBases(_mf, _gs); + var bases = CreateBases(ModuleFactory, GlobalScope); _cls.SetBases(bases); - if (_model.GenericParameterValues.Length > 0) { + if (Model.GenericParameterValues.Length > 0) { _cls.StoreGenericParameters( _cls, _cls.GenericParameters.Keys.ToArray(), - _model.GenericParameterValues.ToDictionary( + Model.GenericParameterValues.ToDictionary( k => _cls.GenericParameters.Keys.First(x => x == k.Name), - v => _mf.ConstructType(v.Type) + v => ModuleFactory.ConstructType(v.Type) ) ); } - var all = _model.Classes.Concat(_model.Properties).Concat(_model.Methods).Concat(_model.Fields).ToArray(); - foreach (var m in all) { - _cls.AddMember(m.Name, m.CreateDeclaration(_mf, _cls, _gs), false); - } - foreach (var m in all) { - m.CreateContent(); + var allMemberModels = Model.Classes + .Concat(Model.Properties) + .Concat(Model.Methods) + .Concat(Model.Fields) + .ToArray(); + + foreach (var model in allMemberModels) { + _cls.AddMember(model.Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _cls), false); } - + ReleaseModel(); } private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { - var ntBases = _model.NamedTupleBases.Select(ntb => { - var n = ntb.CreateDeclaration(mf, _cls, gs); - ntb.CreateContent(); - return n; - }).OfType().ToArray(); + var ntBases = Model.NamedTupleBases + .Select(ntb => MemberFactory.CreateMember(ntb, ModuleFactory, GlobalScope, _cls)) + .OfType() + .ToArray(); var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); - var basesNames = Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); + var basesNames = Model.Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); - if (_model.GenericBaseParameters.Length > 0) { + if (Model.GenericBaseParameters.Length > 0) { // Generic class. Need to reconstruct generic base so code can then // create specific types off the generic class. var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); if (genericBase != null) { - var typeVars = _model.GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); + var typeVars = Model.GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); if (typeVars.Length > 0) { var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); @@ -154,6 +153,5 @@ private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) } return bases; } - } } diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs deleted file mode 100644 index 0e54cc73c..000000000 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionOverload.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using Microsoft.Python.Analysis.Caching.Models; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.Analysis.Caching.Lazy { - internal sealed class PythonLazyFunctionOverload: IPythonFunctionOverload { - private OverloadModel _model; - - public PythonLazyFunctionOverload(OverloadModel model) { - _model = model; - } - - public FunctionDefinition FunctionDefinition { get; } - public IPythonClassMember ClassMember { get; } - public string Name { get; } - public string Documentation { get; } - public IReadOnlyList Parameters { get; } - public IMember Call(IArgumentSet args, IPythonType self) { - throw new NotImplementedException(); - } - - public string GetReturnDocumentation(IPythonType self = null) { - throw new NotImplementedException(); - } - - public IMember StaticReturnValue { get; } - } -} diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs index 49434c205..18202d55f 100644 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -27,9 +27,9 @@ internal sealed class PythonLazyFunctionType : PythonLazyType, IP public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) : base(model, mf, gs, declaringType) { var location = new Location(mf.Module, model.IndexSpan.ToSpan()); - _function = new PythonFunctionType(_model.Name, location, declaringType, Documentation); + _function = new PythonFunctionType(Model.Name, location, declaringType, Documentation); - // TODO: restore signature string so hover does not need to restore function + // TODO: restore signature string so hover (tooltip) documentation won't have to restore the function. // parameters and return type just to look at them. for (var i = 0; i < model.Overloads.Length; i++) { var o = new PythonFunctionOverload(_function, location); @@ -51,28 +51,26 @@ public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScop #endregion protected override void EnsureContent() { - if (_model == null) { + if (Model == null) { return; } // DeclareMember inner functions and classes first since function may be returning one of them. - var innerTypes = _model.Classes.Concat(_model.Functions).ToArray(); + var innerTypes = Model.Classes.Concat(Model.Functions).ToArray(); foreach (var model in innerTypes) { - _function.AddMember(Name, model.CreateDeclaration(_mf, _function, _gs), overwrite: true); - } - foreach (var model in innerTypes) { - model.CreateContent(); + _function.AddMember(Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _function), overwrite: true); } - for (var i = 0; i < _model.Overloads.Length; i++) { - var om = _model.Overloads[i]; + for (var i = 0; i < Model.Overloads.Length; i++) { + var om = Model.Overloads[i]; var o = (PythonFunctionOverload)_function.Overloads[i]; - o.SetReturnValue(_mf.ConstructMember(om.ReturnType), true); - o.SetParameters(om.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); + o.SetReturnValue(ModuleFactory.ConstructMember(om.ReturnType), true); + o.SetParameters(om.Parameters.Select(p => ConstructParameter(ModuleFactory, p)).ToArray()); } ReleaseModel(); } + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); } diff --git a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs index da6e011bc..1a832dfb4 100644 --- a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs @@ -15,30 +15,42 @@ using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Caching.Lazy { - internal sealed class PythonLazyPropertyType: PythonTypeWrapper, IPythonPropertyType { - private readonly PythonPropertyType _innerProperty; - private FunctionModel _model; + internal sealed class PythonLazyPropertyType : PythonLazyType, IPythonPropertyType { + private readonly PythonPropertyType _property; - public PythonLazyPropertyType(PythonPropertyType innerProperty) { - _innerProperty = innerProperty; + public PythonLazyPropertyType(PropertyModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) + : base(model, mf, gs, declaringType) { + + var location = new Location(mf.Module, model.IndexSpan.ToSpan()); + _property = new PythonPropertyType(Model.Name, location, Model.Documentation, declaringType, + Model.Attributes.HasFlag(FunctionAttributes.Abstract)); + + // parameters and return type just to look at them. + var o = new PythonFunctionOverload(_property, location); + o.SetDocumentation(Documentation); + _property.AddOverload(o); + + IsReadOnly = model.IsReadOnly; + SetInnerType(_property); } - public IPythonType DeclaringType => _innerProperty.DeclaringType; public FunctionDefinition FunctionDefinition => null; - public string Description => _innerProperty.Description; - public bool IsReadOnly => _innerProperty.IsReadOnly; + public bool IsReadOnly { get; } + public IMember ReturnType { get { EnsureContent(); - return _innerProperty.ReturnType; + return _property.ReturnType; } } - private void EnsureContent() { - _model?.CreateContent(); - _model = null; + + protected override void EnsureContent() { + _property.Getter.SetReturnValue(ModuleFactory.ConstructMember(Model.ReturnType), true); + ReleaseModel(); } } } diff --git a/src/Caching/Impl/Lazy/PythonLazyType.cs b/src/Caching/Impl/Lazy/PythonLazyType.cs index dbe58199c..51f08235f 100644 --- a/src/Caching/Impl/Lazy/PythonLazyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyType.cs @@ -18,22 +18,30 @@ using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis.Caching.Lazy { + /// + /// Represents 'lazy' type that delays creation of its content such as members, + /// function parameters and return types until they are requested. This allows + /// deferred fetching of data from the database, avoiding wholesale restore. + /// internal abstract class PythonLazyType : PythonTypeWrapper where TModel : class { - protected IGlobalScope _gs; - protected ModuleFactory _mf; - protected TModel _model; + protected IGlobalScope GlobalScope { get; private set; } + protected ModuleFactory ModuleFactory { get; private set; } + protected TModel Model { get; private set; } protected PythonLazyType(TModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) { - _model = model ?? throw new ArgumentNullException(nameof(model)); - _mf = mf ?? throw new ArgumentNullException(nameof(mf)); - _gs = gs ?? throw new ArgumentNullException(nameof(gs)); + Model = model ?? throw new ArgumentNullException(nameof(model)); + ModuleFactory = mf ?? throw new ArgumentNullException(nameof(mf)); + GlobalScope = gs ?? throw new ArgumentNullException(nameof(gs)); + DeclaringType = declaringType; } + public IPythonType DeclaringType { get; } + protected abstract void EnsureContent(); protected void ReleaseModel() { - _mf = null; - _gs = null; - _model = null; + ModuleFactory = null; + GlobalScope = null; + Model = null; } } } diff --git a/src/Caching/Impl/Models/CallableModel.cs b/src/Caching/Impl/Models/CallableModel.cs index 1bf169514..239662d3c 100644 --- a/src/Caching/Impl/Models/CallableModel.cs +++ b/src/Caching/Impl/Models/CallableModel.cs @@ -82,7 +82,6 @@ protected CallableModel(IPythonType callable, IServiceContainer services) { } } } - protected override IEnumerable GetMemberModels() => Classes.Concat(Functions); } } diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index 97f1b219e..4c670f504 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -22,7 +22,6 @@ using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; -using Microsoft.Python.Parsing; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global // ReSharper disable MemberCanBePrivate.Global @@ -49,7 +48,6 @@ internal sealed class ClassModel : MemberModel { public GenericParameterValueModel[] GenericParameterValues { get; set; } [NonSerialized] private readonly ReentrancyGuard _processing = new ReentrancyGuard(); - [NonSerialized] private PythonClassType _cls; public ClassModel() { } // For de-serializer from JSON @@ -132,47 +130,6 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { .Select(p => new GenericParameterValueModel { Name = p.Key, Type = p.Value.GetPersistentQualifiedName(services) }) .ToArray(); } - - /// - /// Restores class from its model for declarations. The class may not be fully constructed - /// yet: method overloads and return types of methods may be missing. - /// - protected override IMember DeclareMember(IPythonType declaringType) { - if (_cls == null) { - _cls = new PythonClassType(Name, new Location(_mf.Module, IndexSpan.ToSpan())); - _cls.SetDocumentation(Documentation); - } - return _cls; - } - - /// - /// Populates class with members. - /// - protected override void PopulateMember() { - var bases = CreateBases(_mf, _gs); - _cls.SetBases(bases); - - if (GenericParameterValues.Length > 0) { - _cls.StoreGenericParameters( - _cls, - _cls.GenericParameters.Keys.ToArray(), - GenericParameterValues.ToDictionary( - k => _cls.GenericParameters.Keys.First(x => x == k.Name), - v => _mf.ConstructType(v.Type) - ) - ); - } - - var all = Classes.Concat(Properties).Concat(Methods).Concat(Fields).ToArray(); - foreach (var m in all) { - _cls.AddMember(m.Name, m.CreateDeclaration(_mf, _cls, _gs), false); - } - foreach (var m in all) { - m.CreateContent(); - } - } - - protected override IEnumerable GetMemberModels() => Classes.Concat(Methods).Concat(Properties).Concat(Fields); } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 1bf35bd39..09597d577 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -156,12 +156,11 @@ private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVaria } return null; } - + public override MemberModel GetModel(string name) { if (_modelCache == null) { - var models = TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); _modelCache = new Dictionary(); - foreach (var m in models) { + foreach (var m in GetMemberModels()) { Debug.Assert(!_modelCache.ContainsKey(m.Name)); _modelCache[m.Name] = m; } @@ -169,11 +168,7 @@ public override MemberModel GetModel(string name) { return _modelCache.TryGetValue(name, out var model) ? model : null; } - public IEnumerable GetMemberNames() - => TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions) - .Concat(Variables)/*.Concat(SubModules)*/.Select(m => m.Name); - - protected override IMember DeclareMember(IPythonType declaringType) => throw new NotImplementedException(); - protected override void PopulateMember() => throw new NotImplementedException(); + protected override IEnumerable GetMemberModels() + => TypeVars.Concat(NamedTuples).Concat(Classes).Concat(Functions).Concat(Variables); } } diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index 2dce55de3..200093dc5 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -18,8 +18,6 @@ using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Specializations.Typing.Types; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching.Models { @@ -42,15 +40,5 @@ public NamedTupleModel(ITypingNamedTupleType nt, IServiceContainer services) { ItemNames = nt.ItemNames.ToArray(); ItemTypes = nt.ItemTypes.Select(t => t.QualifiedName).ToArray(); } - - protected override IMember DeclareMember(IPythonType declaringType) { - if (_namedTuple == null) { - var itemTypes = ItemTypes.Select(_mf.ConstructType).ToArray(); - _namedTuple = new NamedTupleType(Name, ItemNames, itemTypes, _mf.Module, IndexSpan.ToSpan()); - } - return _namedTuple; - } - - protected override void PopulateMember() { } } } diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index 23436820e..d16b09b55 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -14,9 +14,7 @@ // permissions and limitations under the License. using System; -using System.Diagnostics; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; // ReSharper disable AutoPropertyCanBeMadeGetOnly.Global @@ -26,26 +24,15 @@ namespace Microsoft.Python.Analysis.Caching.Models { [Serializable] internal sealed class PropertyModel : CallableModel { public string ReturnType { get; set; } + public bool IsReadOnly { get; set; } + public PropertyModel() { } // For de-serializer from JSON [NonSerialized] private PythonPropertyType _property; public PropertyModel(IPythonPropertyType prop, IServiceContainer services) : base(prop, services) { ReturnType = prop.ReturnType.GetPersistentQualifiedName(services); - } - - protected override IMember DeclareMember(IPythonType declaringType) { - Debug.Assert(_property == null); - _property = new PythonPropertyType(Name, new Location(_mf.Module, IndexSpan.ToSpan()), Documentation, declaringType, (Attributes & FunctionAttributes.Abstract) != 0); - _property.SetDocumentation(Documentation); - return _property; - } - - protected override void PopulateMember() { - var o = new PythonFunctionOverload(_property, _mf.DefaultLocation); - o.SetDocumentation(Documentation); - o.SetReturnValue(_mf.ConstructMember(ReturnType), true); - _property.AddOverload(o); + IsReadOnly = prop.IsReadOnly; } } } diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index 4e54ffe9d..6665a9957 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -17,8 +17,6 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; -using Microsoft.Python.Analysis.Specializations.Typing.Types; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; // ReSharper disable MemberCanBePrivate.Global @@ -44,12 +42,5 @@ public static TypeVarModel FromGeneric(IVariable v, IServiceContainer services) Contravariant = g.Contravariant }; } - - protected override IMember DeclareMember(IPythonType declaringType) - => new GenericTypeParameter(Name, _mf.Module, - Constraints.Select(_mf.ConstructType).ToArray(), - _mf.ConstructType(Bound), Covariant, Contravariant, default); - - protected override void PopulateMember() { } } } diff --git a/src/Caching/Impl/Models/VariableModel.cs b/src/Caching/Impl/Models/VariableModel.cs index 55fd4f10d..6aecbcda4 100644 --- a/src/Caching/Impl/Models/VariableModel.cs +++ b/src/Caching/Impl/Models/VariableModel.cs @@ -48,12 +48,5 @@ internal sealed class VariableModel : MemberModel { IndexSpan = t.Location.IndexSpan.ToModel(), Value = t.GetPersistentQualifiedName(services) }; - - protected override IMember DeclareMember(IPythonType declaringType) { - var m = _mf.ConstructMember(Value) ?? _mf.Module.Interpreter.UnknownType; - return new Variable(Name, m, VariableSource.Declaration, new Location(_mf.Module, IndexSpan?.ToSpan() ?? default)); - } - - protected override void PopulateMember() { } } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index d51d67785..8aaf98cf6 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -18,6 +18,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Python.Analysis.Caching.Lazy; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; @@ -114,11 +115,8 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { return null; } - m = nextModel.CreateDeclaration(this, declaringType, _gs); + m = MemberFactory.CreateMember(nextModel, this, _gs, declaringType); Debug.Assert(m != null); - if (m != null) { - nextModel.CreateContent(); - } if (m is IGenericType gt && typeArgs.Count > 0) { m = gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)); diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index 064d61a9d..e5f8e4256 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -13,12 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Modules; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; @@ -39,11 +36,6 @@ public PythonDbModule(ModuleModel model, string filePath, IServiceContainer serv protected override string LoadContent() => string.Empty; public override string Documentation { get; } - public override IEnumerable GetMemberNames() => GlobalScope.Variables.Names; - public override IMember GetMember(string name) { - ((RestoredGlobalScope)GlobalScope).ReconstructVariable(name); - return base.GetMember(name); - } #region ILocationConverter public override SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(_newLines, index); diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index 5f02fd9ab..da08531e9 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Caching.Lazy; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -25,66 +26,37 @@ namespace Microsoft.Python.Analysis.Caching { internal sealed class RestoredGlobalScope : IGlobalScope { private readonly VariableCollection _scopeVariables = new VariableCollection(); - private ModuleModel _model; // Non-readonly b/c of DEBUG conditional. - private ModuleFactory _factory; // Non-readonly b/c of DEBUG conditional. - private bool _typeVarsCreated; public RestoredGlobalScope(ModuleModel model, IPythonModule module, IServiceContainer services) { - _model = model ?? throw new ArgumentNullException(nameof(model)); Module = module ?? throw new ArgumentNullException(nameof(module)); Name = model.Name; - _factory = new ModuleFactory(_model, Module, this, services); - DeclareVariables(services); + DeclareVariables(model, services); } - public void ReconstructVariable(string name) { - EnsureTypeVars(); - var model = _model.NamedTuples.FirstOrDefault(x => x.Name == name) - ?? _model.Classes.Cast().FirstOrDefault(x => x.Name == name) - ?? _model.Functions.Cast().FirstOrDefault(x => x.Name == name) - //?? _model.SubModules.FirstOrDefault(x => x.Name == name) - ?? _model.Variables.FirstOrDefault(x => x.Name == name); - - model?.CreateContent(); - - // TODO: re-declare __doc__, __name__, etc. -#if !DEBUG - _model = null; - _factory = null; -#endif - } - - private void EnsureTypeVars() { - if (!_typeVarsCreated) { - foreach (var m in _model.TypeVars) { - m.CreateContent(); - } - _typeVarsCreated = true; - } - } - - private void DeclareVariables(IServiceContainer services) { + private void DeclareVariables(ModuleModel model, IServiceContainer services) { // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus first create members so we can find then, then populate them with content. - var mf = new ModuleFactory(_model, Module, this, services); + var mf = new ModuleFactory(model, Module, this, services); // Generics first - foreach (var m in _model.TypeVars) { - _scopeVariables.DeclareVariable(m.Name, m.CreateDeclaration(mf, null, this), VariableSource.Generic, mf.DefaultLocation); + foreach (var m in model.TypeVars) { + var member = MemberFactory.CreateMember(m, mf, this, null); + _scopeVariables.DeclareVariable(m.Name, member, VariableSource.Generic, mf.DefaultLocation); } - var members = _model.NamedTuples - .Concat(_model.Classes).Concat(_model.Functions); //.Concat(_model.SubModules); - foreach (var m in members) { - _scopeVariables.DeclareVariable(m.Name, m.CreateDeclaration(mf, null, this), VariableSource.Declaration, mf.DefaultLocation); + var models = model.NamedTuples + .Concat(model.Classes).Concat(model.Functions); //.Concat(_model.SubModules); + foreach (var m in models) { + var member = MemberFactory.CreateMember(m, mf, this, null); + _scopeVariables.DeclareVariable(m.Name, member, VariableSource.Declaration, mf.DefaultLocation); } - // CreateDeclaration variables in the order of appearance since later variables + // Now variables in the order of appearance since later variables // may use types declared in the preceding ones. - foreach (var vm in _model.Variables.OrderBy(m => m.IndexSpan.Start)) { - var v = (IVariable)vm.CreateDeclaration(mf, null, this); - _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); + foreach (var vm in model.Variables.OrderBy(m => m.IndexSpan.Start)) { + var member = MemberFactory.CreateMember(vm, mf, this, null); + _scopeVariables.DeclareVariable(vm.Name, member, VariableSource.Declaration, mf.DefaultLocation); } } From d613300bb2b62a77829583965c54fc8fb9a71bc9 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Nov 2019 11:40:21 -0800 Subject: [PATCH 010/102] Fix test --- src/Caching/Impl/Lazy/PythonLazyClassType.cs | 6 +++--- src/Caching/Impl/Lazy/PythonLazyFunctionType.cs | 7 ++++--- src/Caching/Impl/Lazy/PythonLazyPropertyType.cs | 2 +- src/Caching/Test/RestoreTests.cs | 13 +++++++------ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs index 2df5fb726..ca49971ba 100644 --- a/src/Caching/Impl/Lazy/PythonLazyClassType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -30,9 +30,9 @@ internal sealed class PythonLazyClassType : PythonLazyType, IPythonC private readonly PythonClassType _cls; public PythonLazyClassType(ClassModel model, ModuleFactory mf, IPythonType declaringType) - : base(model, mf, null, declaringType) { - _cls = new PythonClassType(Name, new Location(mf.Module, model.IndexSpan.ToSpan())); - _cls.SetDocumentation(Documentation); + : base(model, mf, new EmptyGlobalScope(mf.Module), declaringType) { + _cls = new PythonClassType(model.Name, new Location(mf.Module, model.IndexSpan.ToSpan())); + _cls.SetDocumentation(model.Documentation); SetInnerType(_cls); } diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs index 18202d55f..e3be9c81a 100644 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -27,13 +27,15 @@ internal sealed class PythonLazyFunctionType : PythonLazyType, IP public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) : base(model, mf, gs, declaringType) { var location = new Location(mf.Module, model.IndexSpan.ToSpan()); - _function = new PythonFunctionType(Model.Name, location, declaringType, Documentation); + _function = new PythonFunctionType(Model.Name, location, declaringType, model.Documentation); // TODO: restore signature string so hover (tooltip) documentation won't have to restore the function. // parameters and return type just to look at them. for (var i = 0; i < model.Overloads.Length; i++) { + var om = Model.Overloads[i]; var o = new PythonFunctionOverload(_function, location); - o.SetDocumentation(Documentation); + o.SetDocumentation(model.Documentation); + o.SetReturnValueProvider((a, b, c, d) => ModuleFactory.ConstructMember(om.ReturnType)); _function.AddOverload(o); } SetInnerType(_function); @@ -64,7 +66,6 @@ protected override void EnsureContent() { for (var i = 0; i < Model.Overloads.Length; i++) { var om = Model.Overloads[i]; var o = (PythonFunctionOverload)_function.Overloads[i]; - o.SetReturnValue(ModuleFactory.ConstructMember(om.ReturnType), true); o.SetParameters(om.Parameters.Select(p => ConstructParameter(ModuleFactory, p)).ToArray()); } diff --git a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs index 1a832dfb4..8583c46c8 100644 --- a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs @@ -31,7 +31,7 @@ public PythonLazyPropertyType(PropertyModel model, ModuleFactory mf, IGlobalScop // parameters and return type just to look at them. var o = new PythonFunctionOverload(_property, location); - o.SetDocumentation(Documentation); + o.SetDocumentation(model.Documentation); _property.AddOverload(o); IsReadOnly = model.IsReadOnly; diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index 4c930c542..fcce8f160 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -53,8 +53,8 @@ def func2() -> C2: ... await TestData.CreateTestSpecificFileAsync("module2.py", mod2Code); var analysis = await GetAnalysisAsync(code); - var func2 = analysis.Should().HaveVariable("func2").Which; - var analysis2 = ((IPythonFunctionType)func2.Value).DeclaringModule.Analysis; + var f2 = analysis.Should().HaveVariable("func2").Which; + var analysis2 = ((IPythonFunctionType)f2.Value).DeclaringModule.Analysis; var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); Services.AddService(dbs); @@ -64,10 +64,11 @@ def func2() -> C2: ... var doc = Services.GetService().GetDocument(analysis.Document.Uri); analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - analysis.Should().HaveFunction("func2") - .Which.Should().HaveSingleOverload() - .Which.Should().HaveReturnType("C2") - .Which.Should().HaveMember("M1C2") + var func2 = analysis.Should().HaveFunction("func2").Which; + var c2 = func2.Should().HaveSingleOverload() + .Which.Should().HaveReturnType("C2").Which; + + c2.Should().HaveMember("M1C2") .Which.Should().HaveSingleOverload() .Which.Should().HaveReturnType(BuiltinTypeId.Int); } From cdc787562c644edbc38c19ec8c1e884057795487 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Nov 2019 14:24:36 -0800 Subject: [PATCH 011/102] AST library test passed --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 2 +- .../Test/FluentAssertions/MemberAssertions.cs | 4 +- .../Impl/Lazy/PythonLazyFunctionType.cs | 20 +---- src/Caching/Impl/Lazy/PythonLazyOverload.cs | 81 +++++++++++++++++++ src/Caching/Impl/Models/FunctionModel.cs | 3 +- src/Caching/Impl/Models/OverloadModel.cs | 1 + src/Caching/Impl/Models/PropertyModel.cs | 2 - 7 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 src/Caching/Impl/Lazy/PythonLazyOverload.cs diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 8d284822d..bf97086c5 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -125,7 +125,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s #region IPythonType public string Name { get; } - public string QualifiedName => ModuleType == ModuleType.Stub ? $"{Name}(stub)" : Name; + public string QualifiedName => Name; public BuiltinTypeId TypeId => BuiltinTypeId.Module; public bool IsBuiltin => true; public bool IsAbstract => false; diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 1f6b10c08..28d743bf8 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -154,8 +154,8 @@ public void HaveSameMembersAs(IMember other) { otherMember.Should().BeAssignableTo(); } - subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType, $"Type name: {subjectMemberType.Name}"); - //Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); + Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); + subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType); if (subjectMemberType is IPythonClassType subjectClass) { var otherClass = otherMemberType as IPythonClassType; diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs index e3be9c81a..fc6b548ca 100644 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -31,18 +31,14 @@ public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScop // TODO: restore signature string so hover (tooltip) documentation won't have to restore the function. // parameters and return type just to look at them. - for (var i = 0; i < model.Overloads.Length; i++) { - var om = Model.Overloads[i]; - var o = new PythonFunctionOverload(_function, location); - o.SetDocumentation(model.Documentation); - o.SetReturnValueProvider((a, b, c, d) => ModuleFactory.ConstructMember(om.ReturnType)); + foreach (var om in Model.Overloads) { + var o = new PythonLazyOverload(om, mf, _function); _function.AddOverload(o); } SetInnerType(_function); } #region IPythonFunctionType - public IPythonType DeclaringType => _function.DeclaringType; public FunctionDefinition FunctionDefinition => null; public bool IsClassMethod => _function.IsClassMethod; public bool IsStatic => _function.IsStatic; @@ -56,23 +52,11 @@ protected override void EnsureContent() { if (Model == null) { return; } - - // DeclareMember inner functions and classes first since function may be returning one of them. var innerTypes = Model.Classes.Concat(Model.Functions).ToArray(); foreach (var model in innerTypes) { _function.AddMember(Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _function), overwrite: true); } - - for (var i = 0; i < Model.Overloads.Length; i++) { - var om = Model.Overloads[i]; - var o = (PythonFunctionOverload)_function.Overloads[i]; - o.SetParameters(om.Parameters.Select(p => ConstructParameter(ModuleFactory, p)).ToArray()); - } - ReleaseModel(); } - - private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) - => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); } } diff --git a/src/Caching/Impl/Lazy/PythonLazyOverload.cs b/src/Caching/Impl/Lazy/PythonLazyOverload.cs new file mode 100644 index 000000000..ad2bf4d59 --- /dev/null +++ b/src/Caching/Impl/Lazy/PythonLazyOverload.cs @@ -0,0 +1,81 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Caching.Lazy { + internal sealed class PythonLazyOverload: IPythonFunctionOverload { + private readonly PythonFunctionOverload _overload; + private ModuleFactory _mf; + private OverloadModel _model; + + public PythonLazyOverload(OverloadModel model, ModuleFactory mf, IPythonClassMember cm) { + _model = model; + _mf = mf; + + ClassMember = cm; + Documentation = model.Documentation; + + _overload = new PythonFunctionOverload(cm, new Location(mf.Module, default)); + _overload.SetDocumentation(cm.Documentation); + } + + public FunctionDefinition FunctionDefinition => null; + public IPythonClassMember ClassMember { get; } + public string Name => ClassMember.Name; + public string Documentation { get; } + + public IReadOnlyList Parameters { + get { + EnsureContent(); + return _overload.Parameters; + } + } + + public IMember Call(IArgumentSet args, IPythonType self) { + EnsureContent(); + return _overload.Call(args, self); + } + + public string GetReturnDocumentation(IPythonType self = null) { + EnsureContent(); + return _overload.GetReturnDocumentation(self); + } + + public IMember StaticReturnValue { + get { + EnsureContent(); + return _overload.StaticReturnValue; + } + } + + private void EnsureContent() { + if (_model == null) { + return; + } + _overload.SetParameters(_model.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); + _overload.SetReturnValue(_mf.ConstructMember(_model.ReturnType), true); + + _model = null; + _mf = null; + } + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) + => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); + } +} diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 682f011b7..277330f75 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -41,7 +41,8 @@ private static OverloadModel FromOverload(IPythonFunctionOverload o, IServiceCon Kind = p.Kind, DefaultValue = p.DefaultValue.GetPersistentQualifiedName(services), }).ToArray(), - ReturnType = o.StaticReturnValue.GetPersistentQualifiedName(services) + ReturnType = o.StaticReturnValue.GetPersistentQualifiedName(services), + Documentation = o.Documentation }; } } diff --git a/src/Caching/Impl/Models/OverloadModel.cs b/src/Caching/Impl/Models/OverloadModel.cs index 9e03da580..d5a7d5fa9 100644 --- a/src/Caching/Impl/Models/OverloadModel.cs +++ b/src/Caching/Impl/Models/OverloadModel.cs @@ -20,5 +20,6 @@ namespace Microsoft.Python.Analysis.Caching.Models { internal sealed class OverloadModel { public ParameterModel[] Parameters { get; set; } public string ReturnType { get; set; } + public string Documentation { get; set; } } } diff --git a/src/Caching/Impl/Models/PropertyModel.cs b/src/Caching/Impl/Models/PropertyModel.cs index d16b09b55..6108b1f6c 100644 --- a/src/Caching/Impl/Models/PropertyModel.cs +++ b/src/Caching/Impl/Models/PropertyModel.cs @@ -28,8 +28,6 @@ internal sealed class PropertyModel : CallableModel { public PropertyModel() { } // For de-serializer from JSON - [NonSerialized] private PythonPropertyType _property; - public PropertyModel(IPythonPropertyType prop, IServiceContainer services) : base(prop, services) { ReturnType = prop.ReturnType.GetPersistentQualifiedName(services); IsReadOnly = prop.IsReadOnly; From 9e6917ba55b80e909f0f4946815642bad33eeba4 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Nov 2019 15:10:13 -0800 Subject: [PATCH 012/102] Fix more tests --- src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index 95c0a4abc..60c84bb61 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -61,12 +61,15 @@ protected PythonTypeWrapper(BuiltinTypeId builtinTypeId, IPythonModule declaring } protected PythonTypeWrapper() { } - protected void SetInnerType(IPythonType innerType) => _innerType = innerType; + protected void SetInnerType(IPythonType innerType) { + _innerType = innerType; + DeclaringModule = _innerType.DeclaringModule; + } #region IPythonType public virtual string Name => _typeName ?? InnerType.Name; public virtual string QualifiedName => _typeName != null ? $"{DeclaringModule.Name}:{_typeName}" : InnerType.QualifiedName; - public IPythonModule DeclaringModule { get; } + public IPythonModule DeclaringModule { get; private set; } public virtual string Documentation => _documentation ?? InnerType.Documentation; public virtual BuiltinTypeId TypeId => InnerType.TypeId; public virtual PythonMemberType MemberType => InnerType.MemberType; From 960ee87fbbe5ee313a1344c36618a693176c021b Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Nov 2019 16:50:19 -0800 Subject: [PATCH 013/102] Fix generics --- .../Test/FluentAssertions/MemberAssertions.cs | 5 ++- src/Caching/Impl/Lazy/MemberFactory.cs | 2 +- src/Caching/Impl/Lazy/PythonLazyClassType.cs | 41 +++++++++---------- .../Impl/Lazy/PythonLazyFunctionType.cs | 12 ++---- src/Caching/Impl/Lazy/PythonLazyOverload.cs | 16 ++++---- .../Impl/Lazy/PythonLazyPropertyType.cs | 9 ++-- src/Caching/Impl/Lazy/PythonLazyType.cs | 23 +++++++---- src/Caching/Impl/Models/CallableModel.cs | 2 +- 8 files changed, 58 insertions(+), 52 deletions(-) diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 28d743bf8..3af1e65eb 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -154,7 +154,7 @@ public void HaveSameMembersAs(IMember other) { otherMember.Should().BeAssignableTo(); } - Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); + // Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType); if (subjectMemberType is IPythonClassType subjectClass) { @@ -163,7 +163,8 @@ public void HaveSameMembersAs(IMember other) { if (subjectClass is IGenericType gt) { otherClass.Should().BeAssignableTo(); - otherClass.IsGeneric.Should().Be(gt.IsGeneric, $"Class name: {subjectClass.Name}"); + Debug.Assert(otherClass.IsGeneric == gt.IsGeneric); + otherClass.IsGeneric.Should().Be(gt.IsGeneric); } // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. diff --git a/src/Caching/Impl/Lazy/MemberFactory.cs b/src/Caching/Impl/Lazy/MemberFactory.cs index f84f96678..d14ed23dd 100644 --- a/src/Caching/Impl/Lazy/MemberFactory.cs +++ b/src/Caching/Impl/Lazy/MemberFactory.cs @@ -25,7 +25,7 @@ internal static class MemberFactory { public static IMember CreateMember(MemberModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) { switch (model) { case ClassModel cm: - return new PythonLazyClassType(cm, mf, declaringType); + return new PythonLazyClassType(cm, mf, gs, declaringType); case FunctionModel fm: return new PythonLazyFunctionType(fm, mf, gs, declaringType); case PropertyModel pm: diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs index ca49971ba..26750ef6a 100644 --- a/src/Caching/Impl/Lazy/PythonLazyClassType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -29,8 +29,8 @@ namespace Microsoft.Python.Analysis.Caching.Lazy { internal sealed class PythonLazyClassType : PythonLazyType, IPythonClassType { private readonly PythonClassType _cls; - public PythonLazyClassType(ClassModel model, ModuleFactory mf, IPythonType declaringType) - : base(model, mf, new EmptyGlobalScope(mf.Module), declaringType) { + public PythonLazyClassType(ClassModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) + : base(model, mf, gs, declaringType) { _cls = new PythonClassType(model.Name, new Location(mf.Module, model.IndexSpan.ToSpan())); _cls.SetDocumentation(model.Documentation); SetInnerType(_cls); @@ -92,54 +92,53 @@ public IReadOnlyDictionary GenericParameters { } #endregion - protected override void EnsureContent() { - if (Model == null) { - return; - } - - var bases = CreateBases(ModuleFactory, GlobalScope); + protected override void EnsureContent(ClassModel cm) { + var bases = CreateBases(cm, ModuleFactory, GlobalScope); _cls.SetBases(bases); - if (Model.GenericParameterValues.Length > 0) { + if (cm.GenericParameterValues.Length > 0) { _cls.StoreGenericParameters( _cls, _cls.GenericParameters.Keys.ToArray(), - Model.GenericParameterValues.ToDictionary( + cm.GenericParameterValues.ToDictionary( k => _cls.GenericParameters.Keys.First(x => x == k.Name), v => ModuleFactory.ConstructType(v.Type) ) ); } - var allMemberModels = Model.Classes - .Concat(Model.Properties) - .Concat(Model.Methods) - .Concat(Model.Fields) + var allMemberModels = cm.Classes + .Concat(cm.Properties) + .Concat(cm.Methods) + .Concat(cm.Fields) .ToArray(); foreach (var model in allMemberModels) { _cls.AddMember(model.Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _cls), false); } - - ReleaseModel(); } - private IEnumerable CreateBases(ModuleFactory mf, IGlobalScope gs) { - var ntBases = Model.NamedTupleBases + private IEnumerable CreateBases(ClassModel cm, ModuleFactory mf, IGlobalScope gs) { + var ntBases = cm.NamedTupleBases .Select(ntb => MemberFactory.CreateMember(ntb, ModuleFactory, GlobalScope, _cls)) .OfType() .ToArray(); var is3x = mf.Module.Interpreter.LanguageVersion.Is3x(); - var basesNames = Model.Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); + var basesNames = cm.Bases.Select(b => is3x && b == "object" ? null : b).ExcludeDefault().ToArray(); var bases = basesNames.Select(mf.ConstructType).ExcludeDefault().Concat(ntBases).ToArray(); - if (Model.GenericBaseParameters.Length > 0) { + // Make sure base types are realized + foreach (var b in bases.OfType()) { + b.EnsureContent(); + } + + if (cm.GenericBaseParameters.Length > 0) { // Generic class. Need to reconstruct generic base so code can then // create specific types off the generic class. var genericBase = bases.OfType().FirstOrDefault(b => b.Name == "Generic"); if (genericBase != null) { - var typeVars = Model.GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); + var typeVars = cm.GenericBaseParameters.Select(n => gs.Variables[n]?.Value).OfType().ToArray(); //Debug.Assert(typeVars.Length > 0, "Class generic type parameters were not defined in the module during restore"); if (typeVars.Length > 0) { var genericWithParameters = genericBase.CreateSpecificType(new ArgumentSet(typeVars, null, null)); diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs index fc6b548ca..6d53dfa19 100644 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -27,11 +27,11 @@ internal sealed class PythonLazyFunctionType : PythonLazyType, IP public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) : base(model, mf, gs, declaringType) { var location = new Location(mf.Module, model.IndexSpan.ToSpan()); - _function = new PythonFunctionType(Model.Name, location, declaringType, model.Documentation); + _function = new PythonFunctionType(model.Name, location, declaringType, model.Documentation); // TODO: restore signature string so hover (tooltip) documentation won't have to restore the function. // parameters and return type just to look at them. - foreach (var om in Model.Overloads) { + foreach (var om in model.Overloads) { var o = new PythonLazyOverload(om, mf, _function); _function.AddOverload(o); } @@ -48,15 +48,11 @@ public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScop public IReadOnlyList Overloads => _function.Overloads; #endregion - protected override void EnsureContent() { - if (Model == null) { - return; - } - var innerTypes = Model.Classes.Concat(Model.Functions).ToArray(); + protected override void EnsureContent(FunctionModel fm) { + var innerTypes = fm.Classes.Concat(fm.Functions).ToArray(); foreach (var model in innerTypes) { _function.AddMember(Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _function), overwrite: true); } - ReleaseModel(); } } } diff --git a/src/Caching/Impl/Lazy/PythonLazyOverload.cs b/src/Caching/Impl/Lazy/PythonLazyOverload.cs index ad2bf4d59..edce21be7 100644 --- a/src/Caching/Impl/Lazy/PythonLazyOverload.cs +++ b/src/Caching/Impl/Lazy/PythonLazyOverload.cs @@ -21,6 +21,7 @@ namespace Microsoft.Python.Analysis.Caching.Lazy { internal sealed class PythonLazyOverload: IPythonFunctionOverload { + private readonly object _contentLock = new object(); private readonly PythonFunctionOverload _overload; private ModuleFactory _mf; private OverloadModel _model; @@ -66,15 +67,16 @@ public IMember StaticReturnValue { } private void EnsureContent() { - if (_model == null) { - return; + lock (_contentLock) { + if (_model != null) { + _overload.SetParameters(_model.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); + _overload.SetReturnValue(_mf.ConstructMember(_model.ReturnType), true); + _model = null; + _mf = null; + } } - _overload.SetParameters(_model.Parameters.Select(p => ConstructParameter(_mf, p)).ToArray()); - _overload.SetReturnValue(_mf.ConstructMember(_model.ReturnType), true); - - _model = null; - _mf = null; } + private IParameterInfo ConstructParameter(ModuleFactory mf, ParameterModel pm) => new ParameterInfo(pm.Name, mf.ConstructType(pm.Type), pm.Kind, mf.ConstructMember(pm.DefaultValue)); } diff --git a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs index 8583c46c8..def0791b7 100644 --- a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs @@ -26,8 +26,8 @@ public PythonLazyPropertyType(PropertyModel model, ModuleFactory mf, IGlobalScop : base(model, mf, gs, declaringType) { var location = new Location(mf.Module, model.IndexSpan.ToSpan()); - _property = new PythonPropertyType(Model.Name, location, Model.Documentation, declaringType, - Model.Attributes.HasFlag(FunctionAttributes.Abstract)); + _property = new PythonPropertyType(model.Name, location, model.Documentation, declaringType, + model.Attributes.HasFlag(FunctionAttributes.Abstract)); // parameters and return type just to look at them. var o = new PythonFunctionOverload(_property, location); @@ -48,9 +48,8 @@ public IMember ReturnType { } } - protected override void EnsureContent() { - _property.Getter.SetReturnValue(ModuleFactory.ConstructMember(Model.ReturnType), true); - ReleaseModel(); + protected override void EnsureContent(PropertyModel pm) { + _property.Getter.SetReturnValue(ModuleFactory.ConstructMember(pm.ReturnType), true); } } } diff --git a/src/Caching/Impl/Lazy/PythonLazyType.cs b/src/Caching/Impl/Lazy/PythonLazyType.cs index 51f08235f..f248bad87 100644 --- a/src/Caching/Impl/Lazy/PythonLazyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyType.cs @@ -24,12 +24,14 @@ namespace Microsoft.Python.Analysis.Caching.Lazy { /// deferred fetching of data from the database, avoiding wholesale restore. /// internal abstract class PythonLazyType : PythonTypeWrapper where TModel : class { + private readonly object _contentLock = new object(); + private TModel _model; + protected IGlobalScope GlobalScope { get; private set; } protected ModuleFactory ModuleFactory { get; private set; } - protected TModel Model { get; private set; } protected PythonLazyType(TModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) { - Model = model ?? throw new ArgumentNullException(nameof(model)); + _model = model ?? throw new ArgumentNullException(nameof(model)); ModuleFactory = mf ?? throw new ArgumentNullException(nameof(mf)); GlobalScope = gs ?? throw new ArgumentNullException(nameof(gs)); DeclaringType = declaringType; @@ -37,11 +39,18 @@ protected PythonLazyType(TModel model, ModuleFactory mf, IGlobalScope gs, IPytho public IPythonType DeclaringType { get; } - protected abstract void EnsureContent(); - protected void ReleaseModel() { - ModuleFactory = null; - GlobalScope = null; - Model = null; + internal void EnsureContent() { + lock (_contentLock) { + if (_model != null) { + EnsureContent(_model); + + ModuleFactory = null; + GlobalScope = null; + _model = null; + } + } } + + protected abstract void EnsureContent(TModel model); } } diff --git a/src/Caching/Impl/Models/CallableModel.cs b/src/Caching/Impl/Models/CallableModel.cs index 239662d3c..cd258fec9 100644 --- a/src/Caching/Impl/Models/CallableModel.cs +++ b/src/Caching/Impl/Models/CallableModel.cs @@ -62,7 +62,7 @@ protected CallableModel(IPythonType callable, IServiceContainer services) { Id = callable.Name.GetStableHash(); Name = callable.Name; - DeclaringModuleId = callable.DeclaringModule.GetUniqueId(services, AnalysisCachingLevel.Library); + DeclaringModuleId = callable.DeclaringModule.GetUniqueId(services); QualifiedName = callable.QualifiedName; Documentation = callable.Documentation; Classes = classes.ToArray(); From 45ed4eb3cff2bc842c9f21099eb30f0df17d51ee Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Nov 2019 21:18:33 -0800 Subject: [PATCH 014/102] Handle specialization in dependencies --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 25 +++------ .../Impl/Analyzer/PythonAnalyzerSession.cs | 2 - .../Impl/Dependencies/DependencyResolver.cs | 53 ++++++------------- .../Ast/Impl/Dependencies/DependencyVertex.cs | 4 +- .../Impl/Dependencies/IDependencyResolver.cs | 4 +- .../Ast/Test/DependencyResolverTests.cs | 2 - src/Caching/Impl/Models/NamedTupleModel.cs | 2 - src/Caching/Test/RestoreTests.cs | 25 ++++++++- 8 files changed, 49 insertions(+), 68 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index a0584c8eb..b46165700 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -117,7 +117,6 @@ public void RemoveAnalysis(IPythonModule module) { key = new AnalysisModuleKey(module); _analysisEntries.Remove(key); } - _dependencyResolver.Remove(key); } @@ -188,13 +187,13 @@ public async Task ResetAnalyzer() { lock (_syncObj) { _forceGCOnNextSession = true; - _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove); + _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out _); _analysisEntries.Clear(); foreach (var (key, entry) in entriesToPreserve) { _analysisEntries.Add(key, entry); } - _dependencyResolver.RemoveKeys(entriesToRemove.Select(e => e.Key)); + _dependencyResolver.Reset(); } _services.GetService().ReloadAll(); @@ -315,7 +314,6 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray return; } - var foundKeys = ImmutableArray.Empty; foreach (var missingKey in missingKeys) { lock (_syncObj) { if (_analysisEntries.TryGetValue(missingKey, out _)) { @@ -328,21 +326,12 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray var module = moduleResolution.GetOrLoadModule(moduleName); if (module != null && module.ModuleType != ModuleType.Unresolved) { - foundKeys = foundKeys.Add(missingKey); var entry = GetOrCreateAnalysisEntry(module, out _); - _dependencyResolver.TryAddValue(missingKey, entry, entry.IsUserModule, ImmutableArray.Empty); - } - - if (foundKeys.Count > 0) { - foreach (var foundKey in foundKeys) { - PythonAnalyzerEntry entry; - lock (_syncObj) { - if (!_analysisEntries.TryGetValue(foundKey, out entry)) { - continue; - } - } - _dependencyResolver.TryAddValue(foundKey, entry, entry.IsUserModule, ImmutableArray.Empty); - } + _dependencyResolver.TryAddValue(missingKey, + entry, + entry.IsUserModule, + module.ModuleType == ModuleType.Specialized, + ImmutableArray.Empty); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 949b952d9..f47562b64 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -223,8 +223,6 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { lock (_syncObj) { if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { Debug.Assert(_runningTasks == 0); - } else if (!_isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { - _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 5ef5f018a..1c7fa6ce1 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -42,12 +42,12 @@ public int ChangeValue(in TKey key, in TValue value, in bool isRoot, in Immutabl _vertices.Add(default); } - Update(key, value, isRoot, incomingKeys, index); + Update(key, value, isRoot, false, incomingKeys, index); return _version; } } - public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys) { + public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in bool isWalked, in ImmutableArray incomingKeys) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { index = _keys.Count; @@ -57,7 +57,7 @@ public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in Immutabl return _version; } - Update(key, value, isRoot, incomingKeys, index); + Update(key, value, isRoot, isWalked, incomingKeys, index); return _version; } } @@ -98,52 +98,29 @@ public int Remove(in TKey key) { } } - public int RemoveKeys(params TKey[] keys) => RemoveKeys(ImmutableArray.Create(keys)); - - public int RemoveKeys(in ImmutableArray keys) { + /// + /// Removes everything but builtins. + /// + public void Reset() { lock (_syncObj) { - foreach (var key in keys) { - if (_keys.TryGetValue(key, out var index)) { - _vertices[index] = default; - } + if (_vertices.Count > 1) { + _vertices.RemoveRange(1, _vertices.Count - 1); } - var oldKeysReversed = _keys.ToDictionary(kvp => kvp.Value, kvp => kvp.Key); - var oldVertices = new DependencyVertex[_vertices.Count]; - _vertices.CopyTo(oldVertices); - + var kvp = _keys.Count > 0 ? _keys.FirstOrDefault(k => k.Value == 0) : (KeyValuePair?)null; _keys.Clear(); - _vertices.Clear(); - - foreach (var oldVertex in oldVertices) { - if (oldVertex == null) { - continue; - } - - var incomingKeys = oldVertex.Incoming.Select(i => oldKeysReversed[i]); - var key = oldVertex.Key; - var value = oldVertex.Value; - var isRoot = oldVertex.IsRoot; - - if (!_keys.TryGetValue(key, out var index)) { - index = _keys.Count; - _keys[key] = index; - _vertices.Add(default); - } - - Update(key, value, isRoot, incomingKeys, index); + if(kvp != null) { + _keys[kvp.Value.Key] = 0; } - return _version; + _version++; } } - private void Update(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys, in int index) { + private void Update(in TKey key, in TValue value, in bool isRoot, in bool isWalked, in ImmutableArray incomingKeys, in int index) { var version = Interlocked.Increment(ref _version); - var incoming = EnsureKeys(index, incomingKeys, version); - - _vertices[index] = new DependencyVertex(key, value, isRoot, incoming, version, index); + _vertices[index] = new DependencyVertex(key, value, isRoot, isWalked, incoming, version, index); _keys[key] = index; } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index ddbf9160d..672add108 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -51,7 +51,7 @@ public DependencyVertex(DependencyVertex oldVertex, int version, b _state = !isNew && oldVertex.IsWalked ? (int)State.ChangedOutgoing : (int)State.New; } - public DependencyVertex(TKey key, TValue value, bool isRoot, ImmutableArray incoming, int version, int index) { + public DependencyVertex(TKey key, TValue value, bool isRoot, bool isWalked, ImmutableArray incoming, int version, int index) { Key = key; Value = value; IsRoot = isRoot; @@ -59,7 +59,7 @@ public DependencyVertex(TKey key, TValue value, bool isRoot, ImmutableArray Index = index; Incoming = incoming; - _state = (int)State.New; + _state = isWalked ? (int)State.Walked : (int)State.New; } public bool ContainsOutgoing(int index) => _outgoing != null && _outgoing.Contains(index); diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index 3816803e3..e6fc4ead8 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -23,10 +23,10 @@ namespace Microsoft.Python.Analysis.Dependencies { /// concurrently. /// internal interface IDependencyResolver { - int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); + int TryAddValue(in TKey key, in TValue value, in bool isRoot, in bool isWalked, in ImmutableArray incomingKeys); int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); int Remove(in TKey key); - int RemoveKeys(in ImmutableArray keys); + void Reset(); IDependencyChainWalker CreateWalker(); bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDependencyChainWalker walker); diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index 92effbbca..1ffd9e5b3 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -609,9 +609,7 @@ public async Task ChangeValue_RemoveKeys() { node.MarkWalked(); node.MoveNext(); - resolver.RemoveKeys("B", "D"); walker = resolver.CreateWalker(); - walker.MissingKeys.Should().Equal("B", "D"); node = await walker.GetNextAsync(default); node.Value.Should().Be("C:D"); diff --git a/src/Caching/Impl/Models/NamedTupleModel.cs b/src/Caching/Impl/Models/NamedTupleModel.cs index 200093dc5..ae473d30a 100644 --- a/src/Caching/Impl/Models/NamedTupleModel.cs +++ b/src/Caching/Impl/Models/NamedTupleModel.cs @@ -27,8 +27,6 @@ internal sealed class NamedTupleModel: MemberModel { public string[] ItemNames { get; set; } public string[] ItemTypes { get; set; } - [NonSerialized] private NamedTupleType _namedTuple; - public NamedTupleModel() { } // For de-serializer from JSON public NamedTupleModel(ITypingNamedTupleType nt, IServiceContainer services) { diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index fcce8f160..1ba6b0078 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -17,12 +17,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; -using NSubstitute; using TestUtilities; namespace Microsoft.Python.Analysis.Caching.Tests { @@ -72,5 +70,28 @@ def func2() -> C2: ... .Which.Should().HaveSingleOverload() .Which.Should().HaveReturnType(BuiltinTypeId.Int); } + + [TestMethod, Priority(0)] + public async Task Sys() { + const string code = @" +import sys +x = sys.api_version +"; + var analysis = await GetAnalysisAsync(code); + + var sys = analysis.Should().HaveVariable("sys").Which; + var sysAnalysis = ((IPythonModule)sys.Value).Analysis; + + var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); + Services.AddService(dbs); + await dbs.StoreModuleAnalysisAsync(sysAnalysis, CancellationToken.None); + + await Services.GetService().ResetAnalyzer(); + var doc = Services.GetService().GetDocument(analysis.Document.Uri); + analysis = await doc.GetAnalysisAsync(Timeout.Infinite); + + analysis.Should().HaveVariable("x") + .Which.Should().HaveType(BuiltinTypeId.Int); + } } } From 68b2f3be6225fc1cd5edfab4238f6010cac4f145 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 10:12:24 -0800 Subject: [PATCH 015/102] Merge issues --- src/Caching/Impl/ModuleFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 8cb974af5..cb61cb6fe 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -59,7 +59,7 @@ public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs, I } public IPythonType ConstructType(string qualifiedName) - => ConstructMember(qualifiedName)?.GetPythonType() ?? Module.Interpreter.UnknownType; + => ConstructMember(qualifiedName)?.GetPythonType(); public IMember ConstructMember(string qualifiedName) { // Determine module name, member chain and if this is an instance. From e68e1b6c13c9156a6c530e1d5b6d356e8e11bb2d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 10:39:57 -0800 Subject: [PATCH 016/102] Fix os test (posix stub) --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 2 +- .../Ast/Test/FluentAssertions/MemberAssertions.cs | 4 ++-- src/Caching/Impl/ModuleFactory.cs | 6 ++++++ src/Caching/Impl/QualifiedNameParts.cs | 2 ++ src/Caching/Impl/TypeNames.cs | 9 +++++++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index c98d0933c..68fc6256b 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -125,7 +125,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s #region IPythonType public string Name { get; } - public string QualifiedName => Name; + public string QualifiedName => ModuleType == ModuleType.Stub ? $"{Name}(stub)" : Name; public BuiltinTypeId TypeId => BuiltinTypeId.Module; public bool IsBuiltin => true; public bool IsAbstract => false; diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 6fe081ef2..3b698ed2b 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -152,8 +152,8 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje .FailWith($"Expected '{GetName(subjectType)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); } - // Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); - actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType); + actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); + Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); if (actualMemberType is IPythonClassType actualClass) { var expectedClass = expectedMemberType as IPythonClassType; diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index cb61cb6fe..202e9ea03 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -145,7 +145,13 @@ private IPythonModule GetModule(QualifiedNameParts parts) { } // If module is loaded, then use it. Otherwise, create DB module but don't restore it just yet. + // If module is a stub, first try regular module, then the stub since with regular modules + // stub data is merged into the module data but there are also standalone stubs like posix. var module = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); + if (module == null && parts.IsStub) { + module = Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName); + } + // If module is not loaded, try database. if (module == null && parts.ModuleId != null && _db != null) { if (!_modulesCache.TryGetValue(parts.ModuleId, out var m)) { if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, ModuleType.Specialized, out var model)) { diff --git a/src/Caching/Impl/QualifiedNameParts.cs b/src/Caching/Impl/QualifiedNameParts.cs index a4d27fae6..e2db7130a 100644 --- a/src/Caching/Impl/QualifiedNameParts.cs +++ b/src/Caching/Impl/QualifiedNameParts.cs @@ -29,6 +29,8 @@ internal struct QualifiedNameParts { public ObjectType ObjectType; /// Module name. public string ModuleName; + /// Indicates if module is a stub. + public bool IsStub; /// Module unique id. public string ModuleId; /// Module member names such as 'A', 'B', 'C' from module:A.B.C. diff --git a/src/Caching/Impl/TypeNames.cs b/src/Caching/Impl/TypeNames.cs index 2ac14b944..a1fd1aa7b 100644 --- a/src/Caching/Impl/TypeNames.cs +++ b/src/Caching/Impl/TypeNames.cs @@ -127,6 +127,15 @@ private static void GetModuleNameAndMembers(string qualifiedName, ref QualifiedN parts.ModuleName = typeName.Substring(0, moduleSeparatorIndex); var memberNamesOffset = parts.ModuleName.Length + 1; parts.MemberNames = GetTypeNames(typeName.Substring(memberNamesOffset), '.'); + + DetermineModuleType(ref parts); + } + + private static void DetermineModuleType(ref QualifiedNameParts parts) { + if (parts.ModuleName.EndsWith("(stub)")) { + parts.ModuleName = parts.ModuleName.Substring(0, parts.ModuleName.Length - 6); + parts.IsStub = true; + } } public static IReadOnlyList GetTypeNames(string qualifiedTypeName, char separator) { From 800e23174db2ee3edbfd2ea12b556914624426ec Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 11:08:26 -0800 Subject: [PATCH 017/102] Fix sys.stdin case --- src/Caching/Impl/ModuleFactory.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 202e9ea03..ec55e4e2c 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -153,11 +153,12 @@ private IPythonModule GetModule(QualifiedNameParts parts) { } // If module is not loaded, try database. if (module == null && parts.ModuleId != null && _db != null) { - if (!_modulesCache.TryGetValue(parts.ModuleId, out var m)) { + if (_modulesCache.TryGetValue(parts.ModuleId, out var m)) { + module = m; + } else { if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, ModuleType.Specialized, out var model)) { // DeclareMember db module, but do not reconstruct the analysis just yet. - _modulesCache[parts.ModuleId] = m = new PythonDbModule(model, model.FilePath, _services); - module = m; + module = _modulesCache[parts.ModuleId] = new PythonDbModule(model, model.FilePath, _services); } } } @@ -169,9 +170,9 @@ private IPythonModule GetModule(QualifiedNameParts parts) { // happens with io which has member with mmap type coming from mmap // stub rather than the primary mmap module. if (module != null) { - return parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(module) : module; + module = parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(module) : module; } - return null; + return module; } } From d49e246c545b80f1cf20883c44eac5eac714d196 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 12:55:35 -0800 Subject: [PATCH 018/102] Better handle circular references, cache models and resolve os.path --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 1 + src/Caching/Impl/ModuleDatabase.cs | 32 ++++++-- src/Caching/Impl/ModuleFactory.cs | 73 +++++++++---------- src/Caching/Impl/PythonDbModule.cs | 8 ++ src/Caching/Test/AnalysisCachingTestBase.cs | 7 +- src/Caching/Test/LibraryModulesTests.cs | 6 +- src/Caching/Test/ReferencesTests.cs | 1 + 7 files changed, 80 insertions(+), 48 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index c76a258dc..fdc260490 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -526,6 +526,7 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainSingleNode _modulesCache = new Dictionary(); private readonly IServiceContainer _services; private readonly ILogger _log; @@ -61,12 +63,9 @@ public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleT } lock (_lock) { - if (FindModuleModelByPath(moduleName, modulePath, moduleType, out var model)) { - return new PythonDbModule(model, modulePath, _services); - } + return FindModuleModelByPath(moduleName, modulePath, moduleType, out var model) + ? RestoreModule(model) : null; } - - return null; } /// @@ -94,6 +93,25 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul return false; } + internal IPythonModule RestoreModule(string moduleName, string uniqueId) { + lock (_lock) { + return FindModuleModelById(moduleName, uniqueId, out var model) + ? RestoreModule(model) : null; + } + } + + private IPythonModule RestoreModule(ModuleModel model) { + lock (_lock) { + if (_modulesCache.TryGetValue(model.UniqueId, out var m)) { + return m; + } + + var dbModule = _modulesCache[model.UniqueId] = new PythonDbModule(model, model.FilePath, _services); + dbModule.Construct(model); + return dbModule; + } + } + private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { var cachingLevel = GetCachingLevel(); if (cachingLevel == AnalysisCachingLevel.None) { @@ -172,8 +190,8 @@ private string FindDatabaseFile(string uniqueId) { public bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, modulePath, moduleType), out model); - public bool FindModuleModelById(string moduleName, string uniqueId, ModuleType moduleType, out ModuleModel model) - => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, uniqueId, moduleType), out model); + public bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) + => TryGetModuleModel(moduleName, FindDatabaseFile(uniqueId), out model); private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel model) { model = null; diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index ec55e4e2c..4f38ee0fa 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -20,11 +20,11 @@ using System.Linq; using Microsoft.Python.Analysis.Caching.Lazy; using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -36,11 +36,6 @@ internal sealed class ModuleFactory { /// For use in tests so missing members will assert. internal static bool EnableMissingMemberAssertions { get; set; } - private static readonly ConcurrentDictionary _modulesCache - = new ConcurrentDictionary(); - - // TODO: better resolve circular references. - private readonly ReentrancyGuard _moduleReentrancy = new ReentrancyGuard(); private readonly ModuleModel _model; private readonly IGlobalScope _gs; private readonly ModuleDatabase _db; @@ -139,41 +134,45 @@ private IPythonModule GetModule(QualifiedNameParts parts) { return Module; } - using (_moduleReentrancy.Push(parts.ModuleName, out var reentered)) { - if (reentered) { - return null; - } + // If module is loaded, then use it. Otherwise, create DB module but don't restore it just yet. + // If module is a stub, first try regular module, then the stub since with regular modules + // stub data is merged into the module data but there are also standalone stubs like posix. + var mres = Module.Interpreter.ModuleResolution; + var tres = Module.Interpreter.TypeshedResolution; - // If module is loaded, then use it. Otherwise, create DB module but don't restore it just yet. - // If module is a stub, first try regular module, then the stub since with regular modules - // stub data is merged into the module data but there are also standalone stubs like posix. - var module = Module.Interpreter.ModuleResolution.GetImportedModule(parts.ModuleName); - if (module == null && parts.IsStub) { - module = Module.Interpreter.TypeshedResolution.GetImportedModule(parts.ModuleName); - } - // If module is not loaded, try database. - if (module == null && parts.ModuleId != null && _db != null) { - if (_modulesCache.TryGetValue(parts.ModuleId, out var m)) { - module = m; - } else { - if (_db.FindModuleModelById(parts.ModuleName, parts.ModuleId, ModuleType.Specialized, out var model)) { - // DeclareMember db module, but do not reconstruct the analysis just yet. - module = _modulesCache[parts.ModuleId] = new PythonDbModule(model, model.FilePath, _services); - } - } - } + var module = mres.GetImportedModule(parts.ModuleName); + if (module == null && parts.IsStub) { + module = tres.GetImportedModule(parts.ModuleName); + } + + // If module is not loaded, try database. + if (module == null) { + var moduleId = parts.ModuleId ?? parts.ModuleName; + module = _db.RestoreModule(parts.ModuleName, moduleId); + } - // Here we do not call GetOrLoad since modules references here must - // either be loaded already since they were required to create - // persistent state from analysis. Also, occasionally types come - // from the stub and the main module was never loaded. This, for example, - // happens with io which has member with mmap type coming from mmap - // stub rather than the primary mmap module. - if (module != null) { - module = parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(module) : module; + if (module == null) { + // Fallback if somehow module is not loaded or missing from the database. + // Try to load it directly and wait a bit hoping for the analysis to complete. + // Example: posix is a stub-only and is not loaded with the database. + // TODO: consider writing stub-only modules to the database as well. + var resolution = parts.IsStub ? tres : mres; + var imports = resolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, Enumerable.Repeat(parts.ModuleName, 1), true); + if (imports is ModuleImport moduleImport) { + module = resolution.GetOrLoadModule(moduleImport.FullName); } - return module; } + + // Here we do not call GetOrLoad since modules references here must + // either be loaded already since they were required to create + // persistent state from analysis. Also, occasionally types come + // from the stub and the main module was never loaded. This, for example, + // happens with io which has member with mmap type coming from mmap + // stub rather than the primary mmap module. + if (module != null) { + module = parts.ObjectType == ObjectType.VariableModule ? new PythonVariableModule(module) : module; + } + return module; } private IMember GetMember(IMember root, IEnumerable memberNames) { diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index e5f8e4256..1eb0e8ae8 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -30,6 +30,14 @@ public PythonDbModule(ModuleModel model, string filePath, IServiceContainer serv Documentation = model.Documentation; _newLines = model.NewLines.Select(nl => new NewLineLocation(nl.EndIndex, nl.Kind)).ToArray(); _fileSize = model.FileSize; + } + + /// + /// Constructs module global scope. This is separate from regular constructor + /// in order to better handle reentrancy due to circular references + /// in the module factory. + /// + public void Construct(ModuleModel model) { GlobalScope = new RestoredGlobalScope(model, this, Services); } diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index e5eb627d2..fec5f70ae 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -56,8 +56,11 @@ protected string GetBaselineFileName(string testName, string suffix = null) ? Path.Combine(BaselineFilesFolder, testName) : Path.Combine(BaselineFilesFolder, testName + suffix), "json"); - internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) - => new PythonDbModule(model, modulePath, Services); + internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { + var dbModule = new PythonDbModule(model, modulePath, Services); + dbModule.Construct(model); + return dbModule; + } internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) { //var json = ToJson(model); diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index 97fc44751..69e3846db 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -50,8 +50,10 @@ public async Task Builtins() { var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); - var dbModule = new PythonDbModule(model, null, Services); - dbModule.Should().HaveSameMembersAs(builtins); + using (var dbModule = new PythonDbModule(model, null, Services)) { + dbModule.Construct(model); + dbModule.Should().HaveSameMembersAs(builtins); + } } [TestMethod, Priority(0)] diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs index 745fca71a..76b9ac803 100644 --- a/src/Caching/Test/ReferencesTests.cs +++ b/src/Caching/Test/ReferencesTests.cs @@ -66,6 +66,7 @@ def methodB2(self): Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { + dbModule.Construct(model); var sum = dbModule.GetMember("sum") as IPythonFunctionType; sum.Should().NotBeNull(); From 013df6a72ad6b5573872c20b377b0a32eb869cc3 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 15:37:48 -0800 Subject: [PATCH 019/102] Improve stub handling --- src/Analysis/Ast/Impl/Analyzer/StubMerger.cs | 12 ++++++++++++ .../Ast/Test/FluentAssertions/MemberAssertions.cs | 7 ++++--- src/Caching/Impl/ModuleFactory.cs | 2 +- src/Caching/Impl/ModuleUniqueId.cs | 11 ++++++++--- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs index 249da5d0b..94f6b910c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs +++ b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs @@ -94,6 +94,18 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT continue; } + // If type comes from another module and stub type comes from that module stub, skip it. + // For example, 'sqlite3.dbapi2' has Date variable with value from 'datetime' module. + // Stub of 'sqlite3.dbapi2' also has Date from 'datetime (stub)'. We want to use + // type from the primary 'datetime' since it already merged its stub and updated + // type location and documentation while 'datetime' stub does not have documentation + // and its location is irrelevant since we don't navigate to stub source. + if (!_eval.Module.Equals(sourceType?.DeclaringModule) && + sourceType?.DeclaringModule.Stub != null && + sourceType.DeclaringModule.Equals(stubType.DeclaringModule.PrimaryModule)) { + continue; + } + TryReplaceMember(v, sourceType, stubType, cancellationToken); } } diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 3b698ed2b..58f7e09aa 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -153,7 +153,7 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje } actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); - Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); + // Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); if (actualMemberType is IPythonClassType actualClass) { var expectedClass = expectedMemberType as IPythonClassType; @@ -161,8 +161,8 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje if (actualClass is IGenericType gt) { expectedClass.Should().BeAssignableTo(); - Debug.Assert(expectedClass.IsGeneric == gt.IsGeneric); - expectedClass.IsGeneric.Should().Be(gt.IsGeneric); + // Debug.Assert(expectedClass.IsGeneric == gt.IsGeneric); + expectedClass.IsGeneric.Should().Be(gt.IsGeneric, $"{expectedClass.Name} is generic."); } // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. @@ -170,6 +170,7 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); } + Debug.Assert(expectedMemberType.Documentation == actualMemberType.Documentation); if (string.IsNullOrEmpty(expectedMemberType.Documentation)) { assertion.ForCondition(string.IsNullOrEmpty(actualMemberType.Documentation)) .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualMemberType.Documentation}'"); diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 4f38ee0fa..f8a0bb209 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -148,7 +148,7 @@ private IPythonModule GetModule(QualifiedNameParts parts) { // If module is not loaded, try database. if (module == null) { var moduleId = parts.ModuleId ?? parts.ModuleName; - module = _db.RestoreModule(parts.ModuleName, moduleId); + module = _db?.RestoreModule(parts.ModuleName, moduleId); } if (module == null) { diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 135cf4ef4..19aaadce5 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -26,9 +26,14 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { - public static string GetUniqueId(this IPythonModule module, - IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) - => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); + public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { + // If module is a standalone stub, permit it. Otherwise redirect to the main module + // since during stub merge types from stub normally become part of the primary module. + if (module.ModuleType == ModuleType.Stub && module.PrimaryModule != null) { + module = module.PrimaryModule; + } + return GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); + } public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { From 5ebcdfeb7c5eb95ea9f31421d1b54b9e0063f869 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 20:27:04 -0800 Subject: [PATCH 020/102] Test fixes --- src/Analysis/Ast/Impl/Analyzer/StubMerger.cs | 29 +++++++++++++++---- .../Test/FluentAssertions/MemberAssertions.cs | 24 +++++++++------ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs index 94f6b910c..fd0d2cb53 100644 --- a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs +++ b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs @@ -75,6 +75,7 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT if (stubType.IsUnknown()) { continue; } + // If stub says 'Any' but we have better type, keep the current type. if (stubType.DeclaringModule is TypingModule && stubType.Name == "Any") { continue; @@ -84,7 +85,7 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT var sourceType = sourceVar?.Value.GetPythonType(); if (sourceVar?.Source == VariableSource.Import && - sourceVar.GetPythonType()?.DeclaringModule.Stub != null) { + sourceVar.GetPythonType()?.DeclaringModule.Stub != null) { // Keep imported types as they are defined in the library. For example, // 'requests' imports NullHandler as 'from logging import NullHandler'. // But 'requests' also declares NullHandler in its stub (but not in the main code) @@ -94,6 +95,8 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT continue; } + var stubPrimaryModule = stubType.DeclaringModule.PrimaryModule; + // If type comes from another module and stub type comes from that module stub, skip it. // For example, 'sqlite3.dbapi2' has Date variable with value from 'datetime' module. // Stub of 'sqlite3.dbapi2' also has Date from 'datetime (stub)'. We want to use @@ -101,11 +104,27 @@ private void TransferTypesFromStub(IDocumentAnalysis stubAnalysis, CancellationT // type location and documentation while 'datetime' stub does not have documentation // and its location is irrelevant since we don't navigate to stub source. if (!_eval.Module.Equals(sourceType?.DeclaringModule) && - sourceType?.DeclaringModule.Stub != null && - sourceType.DeclaringModule.Equals(stubType.DeclaringModule.PrimaryModule)) { + sourceType?.DeclaringModule.Stub != null && + sourceType.DeclaringModule.Equals(stubPrimaryModule)) { continue; } + // If stub type is not from this module stub, redirect type to primary since primary has locations and documentation. + if (sourceType == null && stubPrimaryModule != null && !stubPrimaryModule.Equals(_eval.Module)) { + Debug.Assert(stubType.DeclaringModule.ModuleType == ModuleType.Stub); + switch (stubType) { + case PythonVariableModule vm: + stubType = vm.Module.PrimaryModule ?? stubType; + break; + case IPythonModule mod: + stubType = mod.PrimaryModule ?? stubType; + break; + default: + stubType = stubPrimaryModule.GetMember(v.Name)?.GetPythonType() ?? stubType; + break; + } + } + TryReplaceMember(v, sourceType, stubType, cancellationToken); } } @@ -261,7 +280,7 @@ private void TransferDocumentationAndLocation(IPythonType sourceType, IPythonTyp // Consider that 'email.headregistry' stub has DataHeader declaring 'datetime' // property of type 'datetime' from 'datetime' module. We don't want to modify // datetime type and change it's location to 'email.headregistry'. - if(stubType.DeclaringModule.ModuleType != ModuleType.Stub || stubType.DeclaringModule != _eval.Module.Stub) { + if (stubType.DeclaringModule.ModuleType != ModuleType.Stub || stubType.DeclaringModule != _eval.Module.Stub) { return; } @@ -311,7 +330,7 @@ private void TransferDocumentationAndLocation(IPythonType sourceType, IPythonTyp /// or location of unrelated types such as coming from the base object type. /// private bool IsFromThisModuleOrSubmodules(IPythonType type) { - if(type.IsUnknown()) { + if (type.IsUnknown()) { return false; } var thisModule = _eval.Module; diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 58f7e09aa..3562e6be6 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -21,6 +21,7 @@ using FluentAssertions; using FluentAssertions.Execution; using FluentAssertions.Primitives; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -152,8 +153,8 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje .FailWith($"Expected '{GetName(subjectType)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); } - actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); // Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); + actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); if (actualMemberType is IPythonClassType actualClass) { var expectedClass = expectedMemberType as IPythonClassType; @@ -162,7 +163,8 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje if (actualClass is IGenericType gt) { expectedClass.Should().BeAssignableTo(); // Debug.Assert(expectedClass.IsGeneric == gt.IsGeneric); - expectedClass.IsGeneric.Should().Be(gt.IsGeneric, $"{expectedClass.Name} is generic."); + // https://github.com/microsoft/python-language-server/issues/1753 + // expectedClass.IsGeneric.Should().Be(gt.IsGeneric, $"{expectedClass.Name} is generic"); } // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. @@ -170,13 +172,17 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); } - Debug.Assert(expectedMemberType.Documentation == actualMemberType.Documentation); - if (string.IsNullOrEmpty(expectedMemberType.Documentation)) { - assertion.ForCondition(string.IsNullOrEmpty(actualMemberType.Documentation)) - .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualMemberType.Documentation}'"); - } else { - assertion.ForCondition(actualMemberType.Documentation.EqualsOrdinal(expectedMemberType.Documentation)) - .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualMemberType.Documentation}'"); + // Allow documentation replacement from primary + // https://github.com/microsoft/python-language-server/issues/1753 + if (expectedMemberType.DeclaringModule.ModuleType != ModuleType.Stub) { + // Debug.Assert(expectedMemberType.Documentation == actualMemberType.Documentation); + if (string.IsNullOrEmpty(expectedMemberType.Documentation)) { + assertion.ForCondition(string.IsNullOrEmpty(actualMemberType.Documentation)) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualMemberType.Documentation}'"); + } else { + assertion.ForCondition(actualMemberType.Documentation.EqualsOrdinal(expectedMemberType.Documentation)) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualMemberType.Documentation}'"); + } } switch (actualMemberType.MemberType) { From 3fff06f760087946d59180e1326972f61c8fd958 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 Nov 2019 23:36:07 -0800 Subject: [PATCH 021/102] Test updates --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 6 +- .../Modules/Definitions/IModuleResolution.cs | 7 +++ .../Resolution/MainModuleResolution.cs | 2 +- .../Modules/Resolution/TypeshedResolution.cs | 9 +++ .../Ast/Test/DependencyResolverTests.cs | 51 ----------------- .../Test/FluentAssertions/MemberAssertions.cs | 2 +- src/Caching/Impl/Lazy/PythonLazyClassType.cs | 1 - src/Caching/Impl/Models/ClassModel.cs | 4 +- src/Caching/Impl/PythonDbModule.cs | 4 +- src/Caching/Impl/RestoredGlobalScope.cs | 3 +- src/Caching/Test/AnalysisCachingTestBase.cs | 12 ++-- src/Caching/Test/ClassesTests.cs | 13 +---- src/Caching/Test/CoreTests.cs | 6 +- .../Test/Files/ClassOwnDocumentation.json | 30 ++++++---- src/Caching/Test/Files/NestedClasses.json | 57 +++++++++++++------ src/Caching/Test/Files/SmokeTest.json | 43 ++++++++++---- src/Caching/Test/Files/VersionHandling3.json | 22 ++++--- src/Caching/Test/LibraryModulesTests.cs | 4 +- src/Caching/Test/ReferencesTests.cs | 6 +- src/Caching/Test/RestoreTests.cs | 24 +++++--- 20 files changed, 168 insertions(+), 138 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index fdc260490..002394855 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -546,8 +546,10 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainSingleNode Task ReloadAsync(CancellationToken token = default); + + /// + /// Returns collection of all currently imported modules. + /// + /// + IEnumerable GetImportedModules(CancellationToken cancellationToken = default); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 08a6ba850..d4eea39c8 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -55,7 +55,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr public IBuiltinsPythonModule BuiltinsModule { get; private set; } - public IEnumerable GetImportedModules(CancellationToken cancellationToken) { + public IEnumerable GetImportedModules(CancellationToken cancellationToken = default) { foreach (var module in _specialized.Values) { cancellationToken.ThrowIfCancellationRequested(); yield return module; diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index 0c42ccbcd..a5ad74b30 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -81,6 +81,15 @@ private bool TryCreateStubModule(string name, out IPythonModule module) { return false; } + public IEnumerable GetImportedModules(CancellationToken cancellationToken = default) { + foreach (var moduleRef in Modules.Values) { + cancellationToken.ThrowIfCancellationRequested(); + if (moduleRef.Value != null) { + yield return moduleRef.Value; + } + } + } + public Task ReloadAsync(CancellationToken cancellationToken = default) { Modules.Clear(); PathResolver = new PathResolver(Interpreter.LanguageVersion, Root, _typeStubPaths, ImmutableArray.Empty); diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index c2739f25e..7a5693747 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -494,57 +494,6 @@ public async Task ChangeValue_RemoveFromLoop() { walker.Remaining.Should().Be(0); } - [TestMethod] - public async Task ChangeValue_RemoveKeys() { - var resolver = new DependencyResolver(); - resolver.ChangeValue("A", "A:BC", true, "B", "C"); - resolver.ChangeValue("B", "B:C", false, "C"); - resolver.ChangeValue("C", "C:D", false, "D"); - resolver.ChangeValue("D", "D", false); - - var walker = resolver.CreateWalker(); - walker.MissingKeys.Should().BeEmpty(); - var node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("D") - .And.HaveOnlyWalkedDependencies(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C:D") - .And.HaveOnlyWalkedDependencies(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("B:C") - .And.HaveOnlyWalkedDependencies(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BC") - .And.HaveOnlyWalkedDependencies(); - node.MarkWalked(); - node.MoveNext(); - - walker = resolver.CreateWalker(); - - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("C:D") - .And.HaveOnlyWalkedDependencies(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Should().HaveSingleValue("A:BC") - .And.HaveOnlyWalkedDependencies(); - node.MarkWalked(); - node.MoveNext(); - - walker.Remaining.Should().Be(0); - } - [TestMethod] public async Task ChangeValue_Skip() { var resolver = new DependencyResolver(); diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 3562e6be6..acd3fb387 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -175,7 +175,7 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje // Allow documentation replacement from primary // https://github.com/microsoft/python-language-server/issues/1753 if (expectedMemberType.DeclaringModule.ModuleType != ModuleType.Stub) { - // Debug.Assert(expectedMemberType.Documentation == actualMemberType.Documentation); + Debug.Assert(expectedMemberType.Documentation == actualMemberType.Documentation); if (string.IsNullOrEmpty(expectedMemberType.Documentation)) { assertion.ForCondition(string.IsNullOrEmpty(actualMemberType.Documentation)) .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualMemberType.Documentation}'"); diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs index 26750ef6a..3a8e8fc9f 100644 --- a/src/Caching/Impl/Lazy/PythonLazyClassType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index 4c670f504..b7b993403 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -104,9 +104,7 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { DeclaringModuleId = cls.DeclaringModule.GetUniqueId(services); QualifiedName = cls.QualifiedName; IndexSpan = cls.Location.IndexSpan.ToModel(); - - // Only persist documentation from this class, leave bases or __init__ alone. - Documentation = (cls as PythonClassType)?.DocumentationSource == PythonClassType.ClassDocumentationSource.Class ? cls.Documentation : null; + Documentation = cls.Documentation; var ntBases = cls.Bases.OfType().ToArray(); NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b, services)).ToArray(); diff --git a/src/Caching/Impl/PythonDbModule.cs b/src/Caching/Impl/PythonDbModule.cs index 1eb0e8ae8..471871ef8 100644 --- a/src/Caching/Impl/PythonDbModule.cs +++ b/src/Caching/Impl/PythonDbModule.cs @@ -38,7 +38,9 @@ public PythonDbModule(ModuleModel model, string filePath, IServiceContainer serv /// in the module factory. /// public void Construct(ModuleModel model) { - GlobalScope = new RestoredGlobalScope(model, this, Services); + var rs = new RestoredGlobalScope(model, this, Services); + GlobalScope = rs; + rs.Construct(model, Services); } protected override string LoadContent() => string.Empty; diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index 9a04ca0dd..a9db371e3 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -30,10 +30,9 @@ internal sealed class RestoredGlobalScope : IGlobalScope { public RestoredGlobalScope(ModuleModel model, IPythonModule module, IServiceContainer services) { Module = module ?? throw new ArgumentNullException(nameof(module)); Name = model.Name; - DeclareVariables(model, services); } - private void DeclareVariables(ModuleModel model, IServiceContainer services) { + public void Construct(ModuleModel model, IServiceContainer services) { // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus first create members so we can find then, then populate them with content. diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index fec5f70ae..58bcf518e 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -62,10 +62,7 @@ internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { return dbModule; } - internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonModule m) { - //var json = ToJson(model); - //Baseline.CompareToFile(BaselineFileName, json); - + internal async Task CompareRestoreAsync(ModuleModel model, IPythonModule m) { var analyzer = Services.GetService(); await analyzer.WaitForCompleteAnalysisAsync(); @@ -73,5 +70,12 @@ internal async Task CompareBaselineAndRestoreAsync(ModuleModel model, IPythonMod dbModule.Should().HaveSameMembersAs(m); } } + + internal async Task GetModelAsync(string code) { + var analysis = await GetAnalysisAsync(code); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + model.FilePath = null; + return model; + } } } diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs index 00aeaa9a3..2a520a510 100644 --- a/src/Caching/Test/ClassesTests.cs +++ b/src/Caching/Test/ClassesTests.cs @@ -59,8 +59,7 @@ def methodB2(self): c = B().methodB1() "; - var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var model = await GetModelAsync(code); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } @@ -80,7 +79,7 @@ def _methodB(self): "; var analysis = await GetAnalysisAsync(code); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - await CompareBaselineAndRestoreAsync(model, analysis.Document); + await CompareRestoreAsync(model, analysis.Document); } [TestMethod, Priority(0)] @@ -116,9 +115,6 @@ def func(): analysis.Should().HaveVariable("b").Which.Should().HaveType("B"); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - //var json = ToJson(model); - //Baseline.CompareToFile(BaselineFileName, json); - using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); } @@ -143,8 +139,6 @@ def value(self): "; var analysis = await GetAnalysisAsync(code); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - //var json = ToJson(model); - //Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); @@ -162,8 +156,7 @@ def __init__(self): '''__init__ doc''' return "; - var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var model = await GetModelAsync(code); var json = ToJson(model); // In JSON, class A should have 'class A doc' documentation while B should have none. Baseline.CompareToFile(BaselineFileName, json); diff --git a/src/Caching/Test/CoreTests.cs b/src/Caching/Test/CoreTests.cs index f1bea1eca..a91845f56 100644 --- a/src/Caching/Test/CoreTests.cs +++ b/src/Caching/Test/CoreTests.cs @@ -58,8 +58,7 @@ def func(): c = C() "; - var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + var model = await GetModelAsync(code); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } @@ -107,6 +106,7 @@ def func(a): ... .Which.Should().HaveParameters(is3x ? new[] { "a", "b", "c" } : new[] { "a" }); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); + model.FilePath = null; var json = ToJson(model); Baseline.CompareToFile(GetBaselineFileNameWithSuffix(is3x ? "3" : "2"), json); } @@ -135,8 +135,6 @@ def func(): "; var analysis = await GetAnalysisAsync(code, PythonVersions.Required_Python38X); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - //var json = ToJson(model); - //Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = CreateDbModule(model, analysis.Document.FilePath)) { dbModule.Should().HaveSameMembersAs(analysis.Document); diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json index 647537712..0c18281c0 100644 --- a/src/Caching/Test/Files/ClassOwnDocumentation.json +++ b/src/Caching/Test/Files/ClassOwnDocumentation.json @@ -1,5 +1,6 @@ { - "UniqueId": "module", + "UniqueId": "module.0", + "FilePath": null, "Documentation": "", "Functions": [], "Variables": [ @@ -7,6 +8,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -17,6 +19,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -27,6 +30,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -37,6 +41,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -47,6 +52,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -57,6 +63,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -67,6 +74,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -77,6 +85,7 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", + "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -99,6 +108,7 @@ "GenericParameterValues": [], "Id": 778, "Name": "A", + "DeclaringModuleId": "module.0", "QualifiedName": "module:A", "IndexSpan": { "Start": 8, @@ -106,9 +116,9 @@ } }, { - "Documentation": null, + "Documentation": "class A doc", "Bases": [ - "t:module:A", + "t:module:A$module.0", "t:object" ], "NamedTupleBases": [], @@ -119,12 +129,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B", + "Type": "t:module:B$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": null + "ReturnType": null, + "Documentation": null } ], "Documentation": null, @@ -133,6 +144,7 @@ "Functions": [], "Id": 965872103, "Name": "__init__", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B.__init__", "IndexSpan": { "Start": 58, @@ -147,6 +159,7 @@ "GenericParameterValues": [], "Id": 779, "Name": "B", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B", "IndexSpan": { "Start": 43, @@ -191,12 +204,9 @@ } ], "FileSize": 115, - "Imports": [], - "FromImports": [], - "StubImports": [], - "StubFromImports": [], - "Id": -2131035837, + "Id": 773962309, "Name": "module", + "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json index efb7c122d..0092fc802 100644 --- a/src/Caching/Test/Files/NestedClasses.json +++ b/src/Caching/Test/Files/NestedClasses.json @@ -1,5 +1,6 @@ { - "UniqueId": "module", + "UniqueId": "module.0", + "FilePath": null, "Documentation": "", "Functions": [], "Variables": [ @@ -7,6 +8,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -17,6 +19,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -27,6 +30,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -37,6 +41,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -47,6 +52,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -57,6 +63,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -67,6 +74,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -77,6 +85,7 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", + "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -87,6 +96,7 @@ "Value": "i:str", "Id": 833, "Name": "x", + "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": { "Start": 2, @@ -94,9 +104,10 @@ } }, { - "Value": "i:module:B.C", + "Value": "i:module:B.C$module.0", "Id": 812, "Name": "c", + "DeclaringModuleId": null, "QualifiedName": "c", "IndexSpan": { "Start": 333, @@ -118,12 +129,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:A", + "Type": "t:module:A$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:bool" + "ReturnType": "i:bool", + "Documentation": null } ], "Documentation": null, @@ -132,6 +144,7 @@ "Functions": [], "Id": -1909501047, "Name": "methodA", + "DeclaringModuleId": "module.0", "QualifiedName": "module:A.methodA", "IndexSpan": { "Start": 33, @@ -146,6 +159,7 @@ "GenericParameterValues": [], "Id": 778, "Name": "A", + "DeclaringModuleId": "module.0", "QualifiedName": "module:A", "IndexSpan": { "Start": 21, @@ -165,12 +179,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B", + "Type": "t:module:B$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:module:B.C" + "ReturnType": "i:module:B.C$module.0", + "Documentation": null } ], "Documentation": null, @@ -179,6 +194,7 @@ "Functions": [], "Id": 935009767, "Name": "methodB1", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B.methodB1", "IndexSpan": { "Start": 235, @@ -191,12 +207,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B", + "Type": "t:module:B$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:int" + "ReturnType": "i:int", + "Documentation": null } ], "Documentation": null, @@ -205,6 +222,7 @@ "Functions": [], "Id": 935009768, "Name": "methodB2", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B.methodB2", "IndexSpan": { "Start": 287, @@ -218,6 +236,7 @@ "Value": "i:int", "Id": 833, "Name": "x", + "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": null } @@ -236,12 +255,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B.C", + "Type": "t:module:B.C$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": null + "ReturnType": null, + "Documentation": null } ], "Documentation": null, @@ -250,6 +270,7 @@ "Functions": [], "Id": 965872103, "Name": "__init__", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B.C.__init__", "IndexSpan": { "Start": 122, @@ -262,12 +283,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B.C", + "Type": "t:module:B.C$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:bool" + "ReturnType": "i:bool", + "Documentation": null } ], "Documentation": null, @@ -276,6 +298,7 @@ "Functions": [], "Id": -1909501045, "Name": "methodC", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B.C.methodC", "IndexSpan": { "Start": 175, @@ -289,6 +312,7 @@ "Value": "i:int", "Id": 834, "Name": "y", + "DeclaringModuleId": null, "QualifiedName": "y", "IndexSpan": null } @@ -298,6 +322,7 @@ "GenericParameterValues": [], "Id": 780, "Name": "C", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B.C", "IndexSpan": { "Start": 106, @@ -309,6 +334,7 @@ "GenericParameterValues": [], "Id": 779, "Name": "B", + "DeclaringModuleId": "module.0", "QualifiedName": "module:B", "IndexSpan": { "Start": 78, @@ -413,12 +439,9 @@ } ], "FileSize": 353, - "Imports": [], - "FromImports": [], - "StubImports": [], - "StubFromImports": [], - "Id": -2131035837, + "Id": 773962309, "Name": "module", + "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json index 3c94c2dc5..73e432301 100644 --- a/src/Caching/Test/Files/SmokeTest.json +++ b/src/Caching/Test/Files/SmokeTest.json @@ -1,12 +1,14 @@ { - "UniqueId": "module", + "UniqueId": "module.0", + "FilePath": null, "Documentation": "", "Functions": [ { "Overloads": [ { "Parameters": [], - "ReturnType": "i:float" + "ReturnType": "i:float", + "Documentation": null } ], "Documentation": null, @@ -15,6 +17,7 @@ "Functions": [], "Id": 24395611, "Name": "func", + "DeclaringModuleId": "module.0", "QualifiedName": "module:func", "IndexSpan": { "Start": 207, @@ -27,6 +30,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -37,6 +41,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -47,6 +52,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -57,6 +63,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -67,6 +74,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -77,6 +85,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -87,6 +96,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -97,6 +107,7 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", + "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -107,6 +118,7 @@ "Value": "i:str", "Id": 833, "Name": "x", + "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": { "Start": 2, @@ -114,9 +126,10 @@ } }, { - "Value": "i:module:C", + "Value": "i:module:C$module.0", "Id": 812, "Name": "c", + "DeclaringModuleId": null, "QualifiedName": "c", "IndexSpan": { "Start": 234, @@ -138,12 +151,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:C", + "Type": "t:module:C$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": null + "ReturnType": null, + "Documentation": null } ], "Documentation": null, @@ -152,6 +166,7 @@ "Functions": [], "Id": 965872103, "Name": "__init__", + "DeclaringModuleId": "module.0", "QualifiedName": "module:C.__init__", "IndexSpan": { "Start": 45, @@ -164,12 +179,13 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:C", + "Type": "t:module:C$module.0", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:float" + "ReturnType": "i:float", + "Documentation": null } ], "Documentation": null, @@ -178,6 +194,7 @@ "Functions": [], "Id": -2139806792, "Name": "method", + "DeclaringModuleId": "module.0", "QualifiedName": "module:C.method", "IndexSpan": { "Start": 100, @@ -188,12 +205,14 @@ "Properties": [ { "ReturnType": "i:int", + "IsReadOnly": true, "Documentation": null, "Attributes": 0, "Classes": [], "Functions": [], "Id": 24690682, "Name": "prop", + "DeclaringModuleId": "module.0", "QualifiedName": "module:C.prop", "IndexSpan": { "Start": 163, @@ -206,6 +225,7 @@ "Value": "i:int", "Id": 833, "Name": "x", + "DeclaringModuleId": null, "QualifiedName": "x", "IndexSpan": null }, @@ -213,6 +233,7 @@ "Value": "i:int", "Id": 834, "Name": "y", + "DeclaringModuleId": null, "QualifiedName": "y", "IndexSpan": null } @@ -222,6 +243,7 @@ "GenericParameterValues": [], "Id": 780, "Name": "C", + "DeclaringModuleId": "module.0", "QualifiedName": "module:C", "IndexSpan": { "Start": 21, @@ -310,12 +332,9 @@ } ], "FileSize": 243, - "Imports": [], - "FromImports": [], - "StubImports": [], - "StubFromImports": [], - "Id": -2131035837, + "Id": 773962309, "Name": "module", + "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json index 1d92c7ff2..9db261eba 100644 --- a/src/Caching/Test/Files/VersionHandling3.json +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -1,5 +1,6 @@ { - "UniqueId": "module", + "UniqueId": "module.0", + "FilePath": null, "Documentation": "", "Functions": [ { @@ -25,7 +26,8 @@ "Kind": 0 } ], - "ReturnType": null + "ReturnType": null, + "Documentation": null } ], "Documentation": null, @@ -34,6 +36,7 @@ "Functions": [], "Id": 24395611, "Name": "func", + "DeclaringModuleId": "module.0", "QualifiedName": "module:func", "IndexSpan": { "Start": 42, @@ -46,6 +49,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -56,6 +60,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -66,6 +71,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -76,6 +82,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -86,6 +93,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -96,6 +104,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -106,6 +115,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -116,6 +126,7 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", + "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -149,12 +160,9 @@ } ], "FileSize": 91, - "Imports": [], - "FromImports": [], - "StubImports": [], - "StubFromImports": [], - "Id": -2131035837, + "Id": 773962309, "Name": "module", + "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index 69e3846db..86ff34494 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -304,7 +304,7 @@ import requests // Verify this looks like a version. new Version(u.Substring(open + 1, u.IndexOf(')') - open - 1)); - await CompareBaselineAndRestoreAsync(model, rq); + await CompareRestoreAsync(model, rq); } private async Task TestModule(string name) { @@ -319,7 +319,7 @@ private async Task TestModule(string name) { var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library); model.Should().NotBeNull($"Module {name} is either not installed or cannot be cached"); - await CompareBaselineAndRestoreAsync(model, m); + await CompareRestoreAsync(model, m); } } } diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs index 76b9ac803..6ad1a48ae 100644 --- a/src/Caching/Test/ReferencesTests.cs +++ b/src/Caching/Test/ReferencesTests.cs @@ -62,8 +62,8 @@ def methodB2(self): "; var analysis = await GetAnalysisAsync(code); var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); - var json = ToJson(model); - Baseline.CompareToFile(BaselineFileName, json); + //var json = ToJson(model); + //Baseline.CompareToFile(BaselineFileName, json); using (var dbModule = new PythonDbModule(model, analysis.Document.FilePath, Services)) { dbModule.Construct(model); @@ -104,7 +104,7 @@ import logging var logging = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("logging"); var model = ModuleModel.FromAnalysis(logging.Analysis, Services, AnalysisCachingLevel.Library); - await CompareBaselineAndRestoreAsync(model, logging); + await CompareRestoreAsync(model, logging); using (var m = CreateDbModule(model, logging.FilePath)) { var critical = m.GetMember("critical") as IPythonFunctionType; diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index 1ba6b0078..127810126 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; @@ -78,13 +79,7 @@ import sys x = sys.api_version "; var analysis = await GetAnalysisAsync(code); - - var sys = analysis.Should().HaveVariable("sys").Which; - var sysAnalysis = ((IPythonModule)sys.Value).Analysis; - - var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); - Services.AddService(dbs); - await dbs.StoreModuleAnalysisAsync(sysAnalysis, CancellationToken.None); + await CreateDatabaseAsync(analysis.Document.Interpreter); await Services.GetService().ResetAnalyzer(); var doc = Services.GetService().GetDocument(analysis.Document.Uri); @@ -93,5 +88,20 @@ import sys analysis.Should().HaveVariable("x") .Which.Should().HaveType(BuiltinTypeId.Int); } + + private async Task CreateDatabaseAsync(IPythonInterpreter interpreter) { + var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); + Services.AddService(dbs); + + var importedModules = interpreter.ModuleResolution.GetImportedModules(); + foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { + await dbs.StoreModuleAnalysisAsync(m.Analysis, CancellationToken.None); + } + + importedModules = interpreter.TypeshedResolution.GetImportedModules(); + foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { + await dbs.StoreModuleAnalysisAsync(m.Analysis, CancellationToken.None); + } + } } } From a928741a2acdd5afe702d346017ade7c740e3a6f Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 10:17:26 -0800 Subject: [PATCH 022/102] Fix deadlock --- src/Caching/Impl/ModuleDatabase.cs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 79fcc00e2..084180048 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -62,10 +62,8 @@ public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleT return null; } - lock (_lock) { - return FindModuleModelByPath(moduleName, modulePath, moduleType, out var model) - ? RestoreModule(model) : null; - } + return FindModuleModelByPath(moduleName, modulePath, moduleType, out var model) + ? RestoreModule(model) : null; } /// @@ -94,22 +92,20 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul } internal IPythonModule RestoreModule(string moduleName, string uniqueId) { - lock (_lock) { - return FindModuleModelById(moduleName, uniqueId, out var model) - ? RestoreModule(model) : null; - } + return FindModuleModelById(moduleName, uniqueId, out var model) + ? RestoreModule(model) : null; } private IPythonModule RestoreModule(ModuleModel model) { + PythonDbModule dbModule; lock (_lock) { if (_modulesCache.TryGetValue(model.UniqueId, out var m)) { return m; } - - var dbModule = _modulesCache[model.UniqueId] = new PythonDbModule(model, model.FilePath, _services); - dbModule.Construct(model); - return dbModule; + dbModule = _modulesCache[model.UniqueId] = new PythonDbModule(model, model.FilePath, _services); } + dbModule.Construct(model); + return dbModule; } private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { @@ -187,7 +183,7 @@ private string FindDatabaseFile(string uniqueId) { return _fs.FileExists(dbPath) ? dbPath : null; } - public bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) + public bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, modulePath, moduleType), out model); public bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) From da64646c59c7000a01b454a4b6fa1b47f9dd9b88 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 11:29:57 -0800 Subject: [PATCH 023/102] Simplify --- src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs | 2 +- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 10 ++++------ src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs | 2 +- .../Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 2 ++ .../Ast/Impl/Dependencies/DependencyResolver.cs | 1 - .../Ast/Impl/Modules/Definitions/IModuleManagement.cs | 4 ---- .../Impl/Modules/Resolution/MainModuleResolution.cs | 6 ++++-- 7 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index 608f2b719..11c9d148c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Analyzer { [DebuggerDisplay("{Name} : {FilePath}")] - public readonly struct AnalysisModuleKey : IEquatable { + internal readonly struct AnalysisModuleKey : IEquatable { public string Name { get; } public string FilePath { get; } public bool IsTypeshed { get; } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 8d1b47b68..f3a9a6e9d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -20,8 +20,6 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -314,20 +312,20 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray return; } - foreach (var missingKey in missingKeys) { + foreach (var key in missingKeys) { lock (_syncObj) { - if (_analysisEntries.TryGetValue(missingKey, out _)) { + if (_analysisEntries.TryGetValue(key, out _)) { continue; } } - var (moduleName, _, isTypeshed) = missingKey; + var (moduleName, _, isTypeshed) = key; var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution; var module = moduleResolution.GetOrLoadModule(moduleName); if (module != null && module.ModuleType != ModuleType.Unresolved) { var entry = GetOrCreateAnalysisEntry(module, out _); - _dependencyResolver.TryAddValue(missingKey, + _dependencyResolver.TryAddValue(key, entry, entry.IsUserModule, module.ModuleType == ModuleType.Specialized, diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 96bb4bf5f..b6fd9c099 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -249,7 +249,7 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { var dependencies = new HashSet(); - if (_bufferVersion > bufferVersion) { + if (_bufferVersion > bufferVersion || module.ModuleType == ModuleType.Specialized) { return dependencies; } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 002394855..52a5d5c72 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -247,6 +247,8 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { lock (_syncObj) { if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { Debug.Assert(_runningTasks == 0); + } else if (!_isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { + //_log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 83316c50b..674eb9fbb 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index 91a034f98..a74e221c3 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -14,8 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; -using System.Threading; using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Types; @@ -88,7 +86,5 @@ public interface IModuleManagement : IModuleResolution { ImmutableArray LibraryPaths { get; } bool SetUserConfiguredPaths(ImmutableArray paths); - - IEnumerable GetImportedModules(CancellationToken cancellationToken); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index d4eea39c8..c08c03ea7 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -99,8 +99,10 @@ protected override IPythonModule CreateModule(string name) { // If there is a stub, make sure it is loaded and attached // First check stub next to the module. - if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { - // If nothing found, try Typeshed. + if (TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { + Analyzer.InvalidateAnalysis(stub); + } else { + // If nothing found, try Typeshed. stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); } From 2b930426b8adacf9c44bd972dfa524c0557e267d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 12:08:39 -0800 Subject: [PATCH 024/102] Properly handle caching level setting --- src/Caching/Impl/Lazy/MemberFactory.cs | 7 ++--- src/Caching/Impl/ModuleDatabase.cs | 19 +++++++------- .../Impl/LanguageServer.Configuration.cs | 26 ++++++++----------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/Caching/Impl/Lazy/MemberFactory.cs b/src/Caching/Impl/Lazy/MemberFactory.cs index d14ed23dd..31e6ebdcf 100644 --- a/src/Caching/Impl/Lazy/MemberFactory.cs +++ b/src/Caching/Impl/Lazy/MemberFactory.cs @@ -23,6 +23,7 @@ namespace Microsoft.Python.Analysis.Caching.Lazy { internal static class MemberFactory { public static IMember CreateMember(MemberModel model, ModuleFactory mf, IGlobalScope gs, IPythonType declaringType) { + var unkType = mf.Module.Interpreter.UnknownType; switch (model) { case ClassModel cm: return new PythonLazyClassType(cm, mf, gs, declaringType); @@ -32,16 +33,16 @@ public static IMember CreateMember(MemberModel model, ModuleFactory mf, IGlobalS return new PythonLazyPropertyType(pm, mf, gs, declaringType); case NamedTupleModel ntm: - var itemTypes = ntm.ItemTypes.Select(mf.ConstructType).ToArray(); + var itemTypes = ntm.ItemTypes.Select(n => mf.ConstructType(n) ?? unkType).ToArray(); return new NamedTupleType(ntm.Name, ntm.ItemNames, itemTypes, mf.Module, ntm.IndexSpan.ToSpan()); case TypeVarModel tvm: return new GenericTypeParameter(tvm.Name, mf.Module, - tvm.Constraints.Select(mf.ConstructType).ToArray(), + tvm.Constraints.Select(n => mf.ConstructType(n) ?? unkType).ToArray(), mf.ConstructType(tvm.Bound), tvm.Covariant, tvm.Contravariant, default); case VariableModel vm: - var m = mf.ConstructMember(vm.Value) ?? mf.Module.Interpreter.UnknownType; + var m = mf.ConstructMember(vm.Value) ?? unkType; return new Variable(vm.Name, m, VariableSource.Declaration, new Location(mf.Module, vm.IndexSpan?.ToSpan() ?? default)); } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 084180048..802eda0a8 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -36,13 +36,15 @@ internal sealed class ModuleDatabase : IModuleDatabaseService { private readonly IServiceContainer _services; private readonly ILogger _log; private readonly IFileSystem _fs; - private readonly AnalysisCachingLevel? _cachingLevel; + private readonly AnalysisCachingLevel _cachingLevel; public ModuleDatabase(IServiceManager sm, string cacheFolder = null, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { _services = sm; _log = _services.GetService(); _fs = _services.GetService(); - _cachingLevel = cachingLevel; + + var optionsProvider = _services.GetService(); + _cachingLevel = optionsProvider?.Options.AnalysisCachingLevel ?? cachingLevel; var cfs = _services.GetService(); CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); @@ -58,7 +60,7 @@ public ModuleDatabase(IServiceManager sm, string cacheFolder = null, AnalysisCac /// Global scope is then can be used to construct module analysis. /// public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleType moduleType) { - if (GetCachingLevel() == AnalysisCachingLevel.None) { + if (_cachingLevel == AnalysisCachingLevel.None) { return null; } @@ -76,7 +78,7 @@ public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationTok /// Determines if module analysis exists in the storage. /// public bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType) { - if (GetCachingLevel() == AnalysisCachingLevel.None) { + if (_cachingLevel == AnalysisCachingLevel.None) { return false; } @@ -109,12 +111,11 @@ private IPythonModule RestoreModule(ModuleModel model) { } private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { - var cachingLevel = GetCachingLevel(); - if (cachingLevel == AnalysisCachingLevel.None) { + if (_cachingLevel == AnalysisCachingLevel.None) { return; } - var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel); + var model = ModuleModel.FromAnalysis(analysis, _services, _cachingLevel); if (model == null) { // Caching level setting does not permit this module to be persisted. return; @@ -157,7 +158,7 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c /// module content (typically file sizes). /// private string FindDatabaseFile(string moduleName, string filePath, ModuleType moduleType) { - var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, GetCachingLevel()); + var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, _cachingLevel); return string.IsNullOrEmpty(uniqueId) ? null : FindDatabaseFile(uniqueId); } @@ -215,7 +216,5 @@ private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel } return false; } - private AnalysisCachingLevel GetCachingLevel() - => _cachingLevel ?? _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; } } diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index e7937bcc7..7c6853091 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -185,21 +185,17 @@ private ImmutableArray GetUserConfiguredPaths(JToken pythonSection) { private const string DefaultCachingLevel = "None"; private AnalysisCachingLevel GetAnalysisCachingLevel(JToken analysisKey) { - // TODO: Remove this one caching is working at any level again. - // https://github.com/microsoft/python-language-server/issues/1758 - return AnalysisCachingLevel.None; - - // var s = GetSetting(analysisKey, "cachingLevel", DefaultCachingLevel); - // - // if (string.IsNullOrWhiteSpace(s) || s.EqualsIgnoreCase("Default")) { - // s = DefaultCachingLevel; - // } - // - // if (s.EqualsIgnoreCase("System")) { - // return AnalysisCachingLevel.System; - // } - // - // return s.EqualsIgnoreCase("Library") ? AnalysisCachingLevel.Library : AnalysisCachingLevel.None; + var s = GetSetting(analysisKey, "cachingLevel", DefaultCachingLevel); + + if (string.IsNullOrWhiteSpace(s) || s.EqualsIgnoreCase("Default")) { + s = DefaultCachingLevel; + } + + if (s.EqualsIgnoreCase("System")) { + return AnalysisCachingLevel.System; + } + + return s.EqualsIgnoreCase("Library") ? AnalysisCachingLevel.Library : AnalysisCachingLevel.None; } } } From 2abe2cf24701188356837eac43a906bfa8f85107 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 12:17:42 -0800 Subject: [PATCH 025/102] Caching level fix --- src/Caching/Impl/ModuleDatabase.cs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 802eda0a8..94f8b99e0 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -36,16 +36,14 @@ internal sealed class ModuleDatabase : IModuleDatabaseService { private readonly IServiceContainer _services; private readonly ILogger _log; private readonly IFileSystem _fs; - private readonly AnalysisCachingLevel _cachingLevel; + private readonly AnalysisCachingLevel _defaultCachingLevel; + private AnalysisCachingLevel? _cachingLevel; - public ModuleDatabase(IServiceManager sm, string cacheFolder = null, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { + public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { _services = sm; _log = _services.GetService(); _fs = _services.GetService(); - - var optionsProvider = _services.GetService(); - _cachingLevel = optionsProvider?.Options.AnalysisCachingLevel ?? cachingLevel; - + _defaultCachingLevel = AnalysisCachingLevel.Library; var cfs = _services.GetService(); CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); sm.AddService(this); @@ -60,7 +58,7 @@ public ModuleDatabase(IServiceManager sm, string cacheFolder = null, AnalysisCac /// Global scope is then can be used to construct module analysis. /// public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleType moduleType) { - if (_cachingLevel == AnalysisCachingLevel.None) { + if (GetCachingLevel() == AnalysisCachingLevel.None) { return null; } @@ -78,7 +76,7 @@ public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationTok /// Determines if module analysis exists in the storage. /// public bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType) { - if (_cachingLevel == AnalysisCachingLevel.None) { + if (GetCachingLevel() == AnalysisCachingLevel.None) { return false; } @@ -111,11 +109,12 @@ private IPythonModule RestoreModule(ModuleModel model) { } private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { - if (_cachingLevel == AnalysisCachingLevel.None) { + var cachingLevel = GetCachingLevel(); + if (cachingLevel == AnalysisCachingLevel.None) { return; } - var model = ModuleModel.FromAnalysis(analysis, _services, _cachingLevel); + var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel); if (model == null) { // Caching level setting does not permit this module to be persisted. return; @@ -158,7 +157,7 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c /// module content (typically file sizes). /// private string FindDatabaseFile(string moduleName, string filePath, ModuleType moduleType) { - var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, _cachingLevel); + var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, moduleType, _services, GetCachingLevel()); return string.IsNullOrEmpty(uniqueId) ? null : FindDatabaseFile(uniqueId); } @@ -216,5 +215,9 @@ private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel } return false; } + private AnalysisCachingLevel GetCachingLevel() + => _cachingLevel + ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) + ?? _defaultCachingLevel; } } From 4711d90fad1cc970fe1ee65d4fd5ff8b12651b00 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 12:47:20 -0800 Subject: [PATCH 026/102] Fix endless analysis --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs | 5 +++-- src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs | 2 +- .../Ast/Impl/Modules/Resolution/MainModuleResolution.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index b6fd9c099..a1e4ac8b6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -188,8 +188,9 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } var dependenciesHashSet = new HashSet(); - foreach (var dependency in analysisDependencies) { - if (dependency != module && (dependency.ModuleType == ModuleType.User && dependency.Analysis.Version < version || dependency.Analysis is EmptyAnalysis)) { + foreach (var dependency in analysisDependencies.ExcludeDefault().Where(d => d.ModuleType != ModuleType.Specialized)) { + if (!dependency.Equals(module) && + (dependency.ModuleType == ModuleType.User && dependency.Analysis.Version < version || dependency.Analysis is EmptyAnalysis)) { dependenciesHashSet.Add(new AnalysisModuleKey(dependency)); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index 672add108..3ffd9b118 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -36,7 +36,7 @@ internal sealed class DependencyVertex { private int _state; private HashSet _outgoing; - private static HashSet _empty = new HashSet(); + private static readonly HashSet _empty = new HashSet(); public DependencyVertex(DependencyVertex oldVertex, int version, bool isNew) { Key = oldVertex.Key; diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index c08c03ea7..d4e9c9ab6 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -91,7 +91,7 @@ protected override IPythonModule CreateModule(string name) { : ModuleType.User; var dbs = GetDbService(); - module = dbs?.RestoreModule(name, moduleImport.ModulePath, ModuleType.Specialized); + module = dbs?.RestoreModule(name, moduleImport.ModulePath, moduleType); if (module != null) { Log?.Log(TraceEventType.Verbose, "Restored from database: ", name); return module; From fdbf219ff29f65a2e93f4a246cea9edba77e7442 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 13:58:14 -0800 Subject: [PATCH 027/102] Fix 'missing keys' message with cached modules --- .../Ast/Impl/Analyzer/CachedAnalysis.cs | 80 +++++++++++++++++++ .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 35 ++++---- .../Ast/Impl/Analyzer/PythonAnalyzerEntry.cs | 19 +++-- .../Impl/Analyzer/PythonAnalyzerSession.cs | 6 +- .../Impl/Dependencies/DependencyResolver.cs | 12 +-- .../Ast/Impl/Dependencies/DependencyVertex.cs | 4 +- .../Dependencies/IDependencyChainWalker.cs | 2 +- .../Impl/Dependencies/IDependencyResolver.cs | 2 +- .../Resolution/MainModuleResolution.cs | 13 ++- 9 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 src/Analysis/Ast/Impl/Analyzer/CachedAnalysis.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/CachedAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/CachedAnalysis.cs new file mode 100644 index 000000000..290c592d8 --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/CachedAnalysis.cs @@ -0,0 +1,80 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer { + /// + /// Analysis of a module restored from database. + /// + internal sealed class CachedAnalysis : IDocumentAnalysis { + public CachedAnalysis(IDocument document, IServiceContainer services) { + Check.ArgumentNotNull(nameof(document), document); + Document = document; + ExpressionEvaluator = new ExpressionEval(services, document, AstUtilities.MakeEmptyAst(document.Uri)); + } + + #region IDocumentAnalysis + /// + /// Analyzed document. + /// + public IDocument Document { get; } + + /// + /// Version of the analysis. Usually matches document version, + /// but can be lower when document or its dependencies were + /// updated since. + /// + public int Version => 0; + + /// + /// Empty AST. + /// + public PythonAst Ast => ExpressionEvaluator.Ast; + + /// + /// Document/module global scope. + /// + public IGlobalScope GlobalScope => Document.GlobalScope; + + /// + /// Expression evaluator used in the analysis. + /// Only supports scope operation since there is no AST + /// when library analysis is complete. + /// + public IExpressionEvaluator ExpressionEvaluator { get; } + + /// + /// Members of the module which are transferred during a star import. null means __all__ was not defined. + /// + public IReadOnlyList StarImportMemberNames => Array.Empty(); + + /// + /// Analysis diagnostics. + /// + public IEnumerable Diagnostics => Enumerable.Empty(); + #endregion + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index f3a9a6e9d..e44a306ea 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -20,6 +20,7 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -255,7 +256,7 @@ private bool TryCreateSession(in int graphVersion, in PythonAnalyzerEntry entry, return false; } - LoadMissingDocuments(entry.Module.Interpreter, walker.MissingKeys); + LoadMissingDocuments(entry.Module.Interpreter, walker); lock (_syncObj) { if (_currentSession == null) { @@ -307,12 +308,8 @@ private PythonAnalyzerSession CreateSession(in IDependencyChainWalker missingKeys) { - if (missingKeys.Count == 0) { - return; - } - - foreach (var key in missingKeys) { + private void LoadMissingDocuments(IPythonInterpreter interpreter, IDependencyChainWalker walker) { + foreach (var key in walker.MissingKeys.ToArray()) { lock (_syncObj) { if (_analysisEntries.TryGetValue(key, out _)) { continue; @@ -325,11 +322,11 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray var module = moduleResolution.GetOrLoadModule(moduleName); if (module != null && module.ModuleType != ModuleType.Unresolved) { var entry = GetOrCreateAnalysisEntry(module, out _); - _dependencyResolver.TryAddValue(key, - entry, - entry.IsUserModule, - module.ModuleType == ModuleType.Specialized, - ImmutableArray.Empty); + if (module.ModuleType == ModuleType.Specialized) { + walker.MissingKeys = walker.MissingKeys.Remove(key); + } else { + _dependencyResolver.TryAddValue(key, entry, entry.IsUserModule, ImmutableArray.Empty); + } } } } @@ -341,12 +338,18 @@ private PythonAnalyzerEntry GetOrCreateAnalysisEntry(IPythonModule module, out A private PythonAnalyzerEntry GetOrCreateAnalysisEntry(IPythonModule module, AnalysisModuleKey key) { lock (_syncObj) { - if (!_analysisEntries.TryGetValue(key, out var entry)) { - var emptyAnalysis = new EmptyAnalysis(_services, (IDocument)module); - entry = new PythonAnalyzerEntry(emptyAnalysis); - _analysisEntries[key] = entry; + if (_analysisEntries.TryGetValue(key, out var entry)) { + return entry; + } + + if (module.ModuleType == ModuleType.Specialized) { + entry = new PythonAnalyzerEntry(new CachedAnalysis((IDocument)module, _services)); + } else { + entry = new PythonAnalyzerEntry(new EmptyAnalysis(_services, (IDocument)module)); _analysisCompleteEvent.Reset(); } + + _analysisEntries[key] = entry; return entry; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index a1e4ac8b6..90ddb0cb9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -84,16 +84,25 @@ public int AnalysisVersion { public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis; - public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) { - _previousAnalysis = emptyAnalysis; - _module = emptyAnalysis.Document; + public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) : this(emptyAnalysis.Document, emptyAnalysis) { + _bufferVersion = -1; + } + + public PythonAnalyzerEntry(CachedAnalysis analysis) : this(analysis.Document, analysis) { + _bufferVersion = 0; + _analysisTcs.SetResult(analysis); + } + + private PythonAnalyzerEntry(IPythonModule module, IDocumentAnalysis analysis) { + _previousAnalysis = analysis; + _module = module; _moduleType = _module.ModuleType; - _bufferVersion = -1; _analysisVersion = 0; _analysisTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); } + public Task GetAnalysisAsync(CancellationToken cancellationToken) => _analysisTcs.Task.WaitAsync(cancellationToken); @@ -189,7 +198,7 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a var dependenciesHashSet = new HashSet(); foreach (var dependency in analysisDependencies.ExcludeDefault().Where(d => d.ModuleType != ModuleType.Specialized)) { - if (!dependency.Equals(module) && + if (!dependency.Equals(module) && (dependency.ModuleType == ModuleType.User && dependency.Analysis.Version < version || dependency.Analysis is EmptyAnalysis)) { dependenciesHashSet.Add(new AnalysisModuleKey(dependency)); } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 52a5d5c72..c82028eb2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -166,10 +166,6 @@ private async Task StartAsync() { totalMilliseconds = Math.Round(totalMilliseconds, 2); (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); - //#if DEBUG - // var notReady = _analyzer.LoadedModules.Where(m => (m.ModuleType == ModuleType.Library || m.ModuleType == ModuleType.Stub) && m.Analysis is EmptyAnalysis).ToArray(); - // Debug.Assert(notReady.Length == 0); - //#endif } } } @@ -248,7 +244,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { Debug.Assert(_runningTasks == 0); } else if (!_isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { - //_log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); + _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 674eb9fbb..ca8be84fc 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -46,12 +46,12 @@ public int ChangeValue(in TKey key, in TValue value, in bool isRoot, in Immutabl _vertices.Add(default); } - Update(key, value, isRoot, false, incomingKeys, index); + Update(key, value, isRoot, incomingKeys, index); return _version; } } - public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in bool isWalked, in ImmutableArray incomingKeys) { + public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { index = _keys.Count; @@ -61,7 +61,7 @@ public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in bool isW return _version; } - Update(key, value, isRoot, isWalked, incomingKeys, index); + Update(key, value, isRoot, incomingKeys, index); return _version; } } @@ -121,10 +121,10 @@ public void Reset() { } } - private void Update(in TKey key, in TValue value, in bool isRoot, in bool isWalked, in ImmutableArray incomingKeys, in int index) { + private void Update(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys, in int index) { var version = Interlocked.Increment(ref _version); var incoming = EnsureKeys(index, incomingKeys, version); - _vertices[index] = new DependencyVertex(key, value, isRoot, isWalked, incoming, version, index); + _vertices[index] = new DependencyVertex(key, value, isRoot, incoming, version, index); _keys[key] = index; } @@ -531,7 +531,7 @@ private sealed class DependencyChainWalker : IDependencyChainWalker _ppc; - public ImmutableArray MissingKeys { get; } + public ImmutableArray MissingKeys { get; set; } public ImmutableArray AffectedValues { get; } public int Version { get; } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index 3ffd9b118..09c1a1cf5 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -51,7 +51,7 @@ public DependencyVertex(DependencyVertex oldVertex, int version, b _state = !isNew && oldVertex.IsWalked ? (int)State.ChangedOutgoing : (int)State.New; } - public DependencyVertex(TKey key, TValue value, bool isRoot, bool isWalked, ImmutableArray incoming, int version, int index) { + public DependencyVertex(TKey key, TValue value, bool isRoot, ImmutableArray incoming, int version, int index) { Key = key; Value = value; IsRoot = isRoot; @@ -59,7 +59,7 @@ public DependencyVertex(TKey key, TValue value, bool isRoot, bool isWalked, Immu Index = index; Incoming = incoming; - _state = isWalked ? (int)State.Walked : (int)State.New; + _state = (int)State.New; } public bool ContainsOutgoing(int index) => _outgoing != null && _outgoing.Contains(index); diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs index 98e68ef0c..4a34f0087 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs @@ -19,7 +19,7 @@ namespace Microsoft.Python.Analysis.Dependencies { internal interface IDependencyChainWalker { - ImmutableArray MissingKeys { get; } + ImmutableArray MissingKeys { get; set; } ImmutableArray AffectedValues { get; } int Version { get; } int Remaining { get; } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index e6fc4ead8..203ea5a71 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Dependencies { /// concurrently. /// internal interface IDependencyResolver { - int TryAddValue(in TKey key, in TValue value, in bool isRoot, in bool isWalked, in ImmutableArray incomingKeys); + int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); int Remove(in TKey key); void Reset(); diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index d4e9c9ab6..9c3682d4d 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -91,10 +91,15 @@ protected override IPythonModule CreateModule(string name) { : ModuleType.User; var dbs = GetDbService(); - module = dbs?.RestoreModule(name, moduleImport.ModulePath, moduleType); - if (module != null) { - Log?.Log(TraceEventType.Verbose, "Restored from database: ", name); - return module; + if (dbs != null) { + var sw = new Stopwatch(); + sw.Start(); + module = dbs.RestoreModule(name, moduleImport.ModulePath, moduleType); + sw.Stop(); + if (module != null) { + Log?.Log(TraceEventType.Verbose, $"Restored from database: {name} in {sw.ElapsedMilliseconds} ms."); + return module; + } } // If there is a stub, make sure it is loaded and attached From 01f763bcdb0a8f6b174cf8219589e4cf12ea1a21 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 14:18:48 -0800 Subject: [PATCH 028/102] Optimize IO --- .../Ast/Impl/Modules/Resolution/MainModuleResolution.cs | 5 ++--- src/Caching/Impl/ModuleDatabase.cs | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 9c3682d4d..150889c86 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -92,8 +92,7 @@ protected override IPythonModule CreateModule(string name) { var dbs = GetDbService(); if (dbs != null) { - var sw = new Stopwatch(); - sw.Start(); + var sw = Stopwatch.StartNew(); module = dbs.RestoreModule(name, moduleImport.ModulePath, moduleType); sw.Stop(); if (module != null) { @@ -107,7 +106,7 @@ protected override IPythonModule CreateModule(string name) { if (TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { Analyzer.InvalidateAnalysis(stub); } else { - // If nothing found, try Typeshed. + // If nothing found, try Typeshed. stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 94f8b99e0..0e131a4b3 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -92,8 +92,12 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul } internal IPythonModule RestoreModule(string moduleName, string uniqueId) { - return FindModuleModelById(moduleName, uniqueId, out var model) - ? RestoreModule(model) : null; + lock (_lock) { + if (_modulesCache.TryGetValue(uniqueId, out var m)) { + return m; + } + } + return FindModuleModelById(moduleName, uniqueId, out var model) ? RestoreModule(model) : null; } private IPythonModule RestoreModule(ModuleModel model) { From d56961967b41a7d770e34a1aa9f038578fb0071a Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 6 Nov 2019 16:24:17 -0800 Subject: [PATCH 029/102] More IO optimizations --- .../Ast/Impl/Analyzer/AnalysisModuleKey.cs | 4 +- .../Resolution/MainModuleResolution.cs | 1 + src/Analysis/Ast/Test/ScrapeTests.cs | 3 +- src/Caching/Impl/ModuleDatabase.cs | 116 ++++++++++-------- src/Core/Impl/IO/PathUtils.cs | 16 +-- 5 files changed, 73 insertions(+), 67 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index 11c9d148c..a92c02906 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Analyzer { [DebuggerDisplay("{Name} : {FilePath}")] - internal readonly struct AnalysisModuleKey : IEquatable { + public readonly struct AnalysisModuleKey : IEquatable { public string Name { get; } public string FilePath { get; } public bool IsTypeshed { get; } @@ -34,7 +34,7 @@ public AnalysisModuleKey(IPythonModule module) : this( module.IsTypeshed, IsNonUserAsDocumentModule(module)) { } - public AnalysisModuleKey(string name, string filePath, bool isTypeshed) + public AnalysisModuleKey(string name, string filePath, bool isTypeshed = false) : this(name, filePath, isTypeshed, false) { } private AnalysisModuleKey(string name, string filePath, bool isTypeshed, bool isNonUserAsDocument) { diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 150889c86..e92faf0ec 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -97,6 +97,7 @@ protected override IPythonModule CreateModule(string name) { sw.Stop(); if (module != null) { Log?.Log(TraceEventType.Verbose, $"Restored from database: {name} in {sw.ElapsedMilliseconds} ms."); + Interpreter.ModuleResolution.SpecializeModule(name, x => module, true); return module; } } diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index d3278475d..e6163e5e4 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -94,7 +94,8 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura var modules = new List(); - foreach (var pyd in PathUtils.EnumerateFiles(fs, dllsDir, "*", recurse: false).Select(f => f.FullName).Where(ModulePath.IsPythonFile)) { + foreach (var pyd in PathUtils.EnumerateFiles(fs, dllsDir, "*", recurse: false) + .Select(f => f.FullName).Where(ModulePath.IsPythonFile)) { var mp = ModulePath.FromFullPath(pyd); if (mp.IsDebug) { continue; diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 0e131a4b3..ed2670534 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -14,12 +14,15 @@ // permissions and limitations under the License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using LiteDB; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -31,7 +34,10 @@ namespace Microsoft.Python.Analysis.Caching { internal sealed class ModuleDatabase : IModuleDatabaseService { private readonly object _lock = new object(); - private static readonly Dictionary _modulesCache = new Dictionary(); + private static readonly Dictionary _modulesCache + = new Dictionary(); + private static readonly ConcurrentDictionary _searchResults + = new ConcurrentDictionary(); private readonly IServiceContainer _services; private readonly ILogger _log; @@ -61,7 +67,6 @@ public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleT if (GetCachingLevel() == AnalysisCachingLevel.None) { return null; } - return FindModuleModelByPath(moduleName, modulePath, moduleType, out var model) ? RestoreModule(model) : null; } @@ -80,10 +85,16 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul return false; } + var key = new AnalysisModuleKey(name, filePath); + if (_searchResults.TryGetValue(key, out var result)) { + return result; + } + for (var retries = 50; retries > 0; --retries) { try { var dbPath = FindDatabaseFile(name, filePath, moduleType); - return !string.IsNullOrEmpty(dbPath); + _searchResults[key] = result = !string.IsNullOrEmpty(dbPath); + return result; } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { Thread.Sleep(10); } @@ -124,35 +135,21 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c return; } - Exception ex = null; - for (var retries = 50; retries > 0; --retries) { - cancellationToken.ThrowIfCancellationRequested(); - try { - if (!_fs.DirectoryExists(CacheFolder)) { - _fs.CreateDirectory(CacheFolder); - } - - cancellationToken.ThrowIfCancellationRequested(); - using (var db = new LiteDatabase(Path.Combine(CacheFolder, $"{model.UniqueId}.db"))) { - var modules = db.GetCollection("modules"); - modules.Upsert(model); - return; - } - } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { - ex = ex1; - Thread.Sleep(10); - } catch (Exception ex2) { - ex = ex2; - break; + WithRetries(() => { + if (!_fs.DirectoryExists(CacheFolder)) { + _fs.CreateDirectory(CacheFolder); } - } + return true; + }, $"Unable to create directory {CacheFolder} for modules cache."); - if (ex != null) { - _log?.Log(System.Diagnostics.TraceEventType.Warning, $"Unable to write analysis of {model.Name} to database. Exception {ex.Message}"); - if (ex.IsCriticalException()) { - throw ex; + WithRetries(() => { + cancellationToken.ThrowIfCancellationRequested(); + using (var db = new LiteDatabase(Path.Combine(CacheFolder, $"{model.UniqueId}.db"))) { + var modules = db.GetCollection("modules"); + modules.Upsert(model); } - } + return true; + }, $"Unable to write analysis of {model.Name} to database."); } /// @@ -187,41 +184,54 @@ private string FindDatabaseFile(string uniqueId) { return _fs.FileExists(dbPath) ? dbPath : null; } - public bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) + private bool FindModuleModelByPath(string moduleName, string modulePath, ModuleType moduleType, out ModuleModel model) => TryGetModuleModel(moduleName, FindDatabaseFile(moduleName, modulePath, moduleType), out model); - public bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) + private bool FindModuleModelById(string moduleName, string uniqueId, out ModuleModel model) => TryGetModuleModel(moduleName, FindDatabaseFile(uniqueId), out model); private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel model) { model = null; - // We don't cache results here. Module resolution service decides when to call in here - // and it is responsible of overall management of the loaded Python modules. - for (var retries = 50; retries > 0; --retries) { - try { - // TODO: make combined db rather than per module? - if (string.IsNullOrEmpty(dbPath)) { - return false; - } - - using (var db = new LiteDatabase(dbPath)) { - if (!db.CollectionExists("modules")) { - return false; - } - - var modules = db.GetCollection("modules"); - model = modules.Find(m => m.Name == moduleName).FirstOrDefault(); - return model != null; - } - } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { - Thread.Sleep(10); - } + + if (string.IsNullOrEmpty(dbPath)) { + return false; } - return false; + + model = WithRetries(() => { + using (var db = new LiteDatabase(dbPath)) { + var modules = db.GetCollection("modules"); + return modules.Find(m => m.Name == moduleName).FirstOrDefault(); + } + }, $"Unable to locate database for module {moduleName}."); + + return model != null; } + private AnalysisCachingLevel GetCachingLevel() => _cachingLevel ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) ?? _defaultCachingLevel; + + private T WithRetries(Func a, string errorMessage) { + Exception ex = null; + for (var retries = 50; retries > 0; --retries) { + try { + return a(); + } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { + ex = ex1; + Thread.Sleep(10); + } catch (Exception ex2) { + ex = ex2; + break; + } + } + if (ex != null) { + _log?.Log(TraceEventType.Warning, $"{errorMessage} Exception: {ex.Message}"); + if (ex.IsCriticalException()) { + throw ex; + } + } + return default; + } } } diff --git a/src/Core/Impl/IO/PathUtils.cs b/src/Core/Impl/IO/PathUtils.cs index ff73cd966..3450a03f8 100644 --- a/src/Core/Impl/IO/PathUtils.cs +++ b/src/Core/Impl/IO/PathUtils.cs @@ -276,10 +276,6 @@ public static string GetFileName(string filePath) { /// /// true to return files within subdirectories. /// - /// - /// true to return full paths for all subdirectories. Otherwise, - /// the relative path from is returned. - /// public static IEnumerable EnumerateFiles(IFileSystem fileSystem, string root, string pattern = "*", bool recurse = true) { root = EnsureEndSeparator(root); @@ -292,13 +288,11 @@ public static IEnumerable EnumerateFiles(IFileSystem fileSystem, stri var fullDir = Path.IsPathRooted(dir) ? dir : root + dir; IFileInfo[] files = null; try { - if (fileSystem.DirectoryExists(fullDir)) { - files = fileSystem.GetDirectoryInfo(fullDir) - .EnumerateFileSystemInfos(pattern, SearchOption.TopDirectoryOnly) - .Where(f => !f.Attributes.HasFlag(FileAttributes.Directory)) - .OfType() - .ToArray(); - } + files = fileSystem.GetDirectoryInfo(fullDir) + .EnumerateFileSystemInfos(pattern, SearchOption.TopDirectoryOnly) + .Where(f => !f.Attributes.HasFlag(FileAttributes.Directory)) + .OfType() + .ToArray(); } catch (UnauthorizedAccessException) { } catch (IOException) { } From 2a64510f60fbb9c2f1816e805677f18004a2b2da Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 7 Nov 2019 09:49:18 -0800 Subject: [PATCH 030/102] Full restore tests --- .../Test/FluentAssertions/MemberAssertions.cs | 15 ++++++++----- src/Caching/Test/RestoreTests.cs | 22 ++++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index acd3fb387..31179af86 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -175,13 +175,16 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje // Allow documentation replacement from primary // https://github.com/microsoft/python-language-server/issues/1753 if (expectedMemberType.DeclaringModule.ModuleType != ModuleType.Stub) { - Debug.Assert(expectedMemberType.Documentation == actualMemberType.Documentation); - if (string.IsNullOrEmpty(expectedMemberType.Documentation)) { - assertion.ForCondition(string.IsNullOrEmpty(actualMemberType.Documentation)) - .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualMemberType.Documentation}'"); + var expectedDoc = expectedMemberType.Documentation?.Trim(); + var actualDoc = actualMemberType.Documentation?.Trim(); + + Debug.Assert(expectedDoc == actualDoc); + if (string.IsNullOrEmpty(expectedDoc)) { + assertion.ForCondition(string.IsNullOrEmpty(actualDoc)) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualDoc}'"); } else { - assertion.ForCondition(actualMemberType.Documentation.EqualsOrdinal(expectedMemberType.Documentation)) - .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualMemberType.Documentation}'"); + assertion.ForCondition(actualDoc.EqualsOrdinal(expectedDoc)) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualDoc}'"); } } diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index 127810126..5cbefbef9 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -73,20 +73,22 @@ def func2() -> C2: ... } [TestMethod, Priority(0)] - public async Task Sys() { - const string code = @" -import sys -x = sys.api_version -"; - var analysis = await GetAnalysisAsync(code); - await CreateDatabaseAsync(analysis.Document.Interpreter); + public Task Sys() => TestModule("sys"); + [TestMethod, Priority(0)] + public Task Numpy() => TestModule("numpy"); + + private async Task TestModule(string name) { + var analysis = await GetAnalysisAsync($"import {name}"); + var sys = analysis.Should().HaveVariable(name).Which; + + await CreateDatabaseAsync(analysis.Document.Interpreter); await Services.GetService().ResetAnalyzer(); var doc = Services.GetService().GetDocument(analysis.Document.Uri); - analysis = await doc.GetAnalysisAsync(Timeout.Infinite); - analysis.Should().HaveVariable("x") - .Which.Should().HaveType(BuiltinTypeId.Int); + var restored = await doc.GetAnalysisAsync(Timeout.Infinite); + var restoredSys = restored.Should().HaveVariable(name).Which; + restoredSys.Value.Should().HaveSameMembersAs(sys.Value); } private async Task CreateDatabaseAsync(IPythonInterpreter interpreter) { From f2bd272eacf82c6358a3d3f334ef3dedd68dbffa Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 7 Nov 2019 18:04:04 -0800 Subject: [PATCH 031/102] Deeper comparison in tests --- src/Analysis/Ast/Impl/Analyzer/StubMerger.cs | 20 ++- .../Analyzer/Symbols/FunctionEvaluator.cs | 2 +- .../Test/FluentAssertions/MemberAssertions.cs | 160 ++++++++++-------- src/Caching/Impl/ModuleFactory.cs | 3 +- src/Caching/Test/RestoreTests.cs | 3 + 5 files changed, 106 insertions(+), 82 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs index fd0d2cb53..54eb2dd29 100644 --- a/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs +++ b/src/Analysis/Ast/Impl/Analyzer/StubMerger.cs @@ -141,8 +141,12 @@ private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType s } break; - case PythonClassType sourceClass: - MergeClass(v, sourceClass, stubType, cancellationToken); + case IPythonClassType sourceClass: + MergeMembers(v, sourceClass, stubType, cancellationToken); + break; + + case PythonFunctionType sourceFunction: + MergeMembers(v, sourceFunction, stubType, cancellationToken); break; case IPythonModule _: @@ -168,11 +172,11 @@ private void TryReplaceMember(IVariable v, IPythonType sourceType, IPythonType s } } - private void MergeClass(IVariable v, IPythonClassType sourceClass, IPythonType stubType, CancellationToken cancellationToken) { + private void MergeMembers(IVariable v, IPythonType sourceType, IPythonType stubType, CancellationToken cancellationToken) { // Transfer documentation first so we get class documentation // that comes from the class definition win over one that may // come from __init__ during the member merge below. - TransferDocumentationAndLocation(sourceClass, stubType); + TransferDocumentationAndLocation(sourceType.GetPythonType(), stubType); // Replace the class entirely since stub members may use generic types // and the class definition is important. We transfer missing members @@ -181,15 +185,15 @@ private void MergeClass(IVariable v, IPythonClassType sourceClass, IPythonType s // First pass: go through source class members and pick those // that are not present in the stub class. - foreach (var name in sourceClass.GetMemberNames().ToArray()) { + foreach (var name in sourceType.GetMemberNames().ToArray()) { cancellationToken.ThrowIfCancellationRequested(); - var sourceMember = sourceClass.GetMember(name); + var sourceMember = sourceType.GetMember(name); if (sourceMember.IsUnknown()) { continue; // Do not add unknowns to the stub. } var sourceMemberType = sourceMember?.GetPythonType(); - if (sourceMemberType is IPythonClassMember cm && cm.DeclaringType != sourceClass) { + if (sourceMemberType is IPythonClassMember cm && cm.DeclaringType != sourceType) { continue; // Only take members from this class and not from bases. } if (!IsFromThisModuleOrSubmodules(sourceMemberType)) { @@ -227,7 +231,7 @@ private void MergeClass(IVariable v, IPythonClassType sourceClass, IPythonType s continue; // Only take members from this class and not from bases. } - var sourceMember = sourceClass.GetMember(name); + var sourceMember = sourceType.GetMember(name); if (sourceMember.IsUnknown()) { continue; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 19e94d2f7..5348dbc2d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -74,7 +74,7 @@ public override void Evaluate() { v => v.GetPythonType() == null && v.GetPythonType() == null) ) { - ((VariableCollection)Eval.CurrentScope.Variables).Clear(); + ((VariableCollection)Eval.CurrentScope.Variables).Clear(); } } } diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 31179af86..491fc0edd 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; @@ -128,92 +129,109 @@ public AndConstraint HaveSameMemberNamesAs(IMember member) { return HaveMembers(((IMemberContainer)member).GetMemberNames(), string.Empty); } + private static readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); + public void HaveSameMembersAs(IMember expected, string because = "", params object[] becauseArgs) { var expectedContainer = expected.Should().BeAssignableTo().Which; + var actualContainer = Subject.GetPythonType(); - var subjectType = Subject.GetPythonType(); - var actualNames = subjectType.GetMemberNames().ToArray(); - var expectedNames = expectedContainer.GetMemberNames().ToArray(); - - var errorMessage = GetAssertCollectionOnlyContainsMessage(actualNames, expectedNames, GetQuotedName(Subject), "member", "members"); + using (_memberGuard.Push(actualContainer, out var reentered)) { + if (reentered) { + return; + } + var actualNames = actualContainer.GetMemberNames().ToArray(); + var expectedNames = expectedContainer.GetMemberNames().ToArray(); - var assertion = Execute.Assertion.BecauseOf(because, becauseArgs); + var errorMessage = GetAssertCollectionOnlyContainsMessage(actualNames, expectedNames, GetQuotedName(Subject), "member", "members"); + var assertion = Execute.Assertion.BecauseOf(because, becauseArgs); - assertion.ForCondition(errorMessage == null).FailWith(errorMessage); + assertion.ForCondition(errorMessage == null).FailWith(errorMessage); - foreach (var n in actualNames.Except(Enumerable.Repeat("__base__", 1))) { - var actualMember = subjectType.GetMember(n); - var expectedMember = expectedContainer.GetMember(n); - var actualMemberType = actualMember.GetPythonType(); - var expectedMemberType = expectedMember.GetPythonType(); + foreach (var n in actualNames.Except(Enumerable.Repeat("__base__", 1))) { + var actualMember = actualContainer.GetMember(n); + var expectedMember = expectedContainer.GetMember(n); - // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. - if (expectedMember is IPythonInstance && !expectedMember.IsUnknown()) { - assertion.ForCondition(actualMember is IPythonInstance) - .FailWith($"Expected '{GetName(subjectType)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); - } + var actualMemberType = actualMember.GetPythonType(); + var expectedMemberType = expectedMember.GetPythonType(); - // Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); - actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); + // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. + if (expectedMember is IPythonInstance && !expectedMember.IsUnknown()) { + Debug.Assert(actualMember is IPythonInstance); + assertion.ForCondition(actualMember is IPythonInstance) + .FailWith($"Expected '{GetName(actualContainer)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); + } - if (actualMemberType is IPythonClassType actualClass) { - var expectedClass = expectedMemberType as IPythonClassType; - expectedClass.Should().NotBeNull(); + // Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); + actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); - if (actualClass is IGenericType gt) { - expectedClass.Should().BeAssignableTo(); - // Debug.Assert(expectedClass.IsGeneric == gt.IsGeneric); - // https://github.com/microsoft/python-language-server/issues/1753 - // expectedClass.IsGeneric.Should().Be(gt.IsGeneric, $"{expectedClass.Name} is generic"); - } + #region Class comparison + if (actualMemberType is IPythonClassType actualClass) { + var expectedClass = expectedMemberType as IPythonClassType; + expectedClass.Should().NotBeNull(); - // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. - //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); - //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); - } + if (actualClass is IGenericType gt) { + expectedClass.Should().BeAssignableTo(); + // Debug.Assert(expectedClass.IsGeneric == gt.IsGeneric); + // https://github.com/microsoft/python-language-server/issues/1753 + // expectedClass.IsGeneric.Should().Be(gt.IsGeneric, $"{expectedClass.Name} is generic"); + } - // Allow documentation replacement from primary - // https://github.com/microsoft/python-language-server/issues/1753 - if (expectedMemberType.DeclaringModule.ModuleType != ModuleType.Stub) { - var expectedDoc = expectedMemberType.Documentation?.Trim(); - var actualDoc = actualMemberType.Documentation?.Trim(); - - Debug.Assert(expectedDoc == actualDoc); - if (string.IsNullOrEmpty(expectedDoc)) { - assertion.ForCondition(string.IsNullOrEmpty(actualDoc)) - .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualDoc}'"); - } else { - assertion.ForCondition(actualDoc.EqualsOrdinal(expectedDoc)) - .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualDoc}'"); + // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. + //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); + //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); } - } - - switch (actualMemberType.MemberType) { - case PythonMemberType.Class: - // Restored collections (like instance of tuple) turn into classes - // rather than into collections with content since we don't track - // collection content in libraries. We don't compare qualified names - // since original module may be source or a stub and that is not - // preserved during restore. - // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); - break; - case PythonMemberType.Function: - case PythonMemberType.Method: - actualMemberType.Should().BeAssignableTo(); - expectedMemberType.Should().BeAssignableTo(); - if (actualMemberType is IPythonFunctionType subjectFunction) { - var otherFunction = (IPythonFunctionType)expectedMemberType; - subjectFunction.Should().HaveSameOverloadsAs(otherFunction); + #endregion + + #region Documentation comparison + // Allow documentation replacement from primary + // https://github.com/microsoft/python-language-server/issues/1753 + if (expectedMemberType.DeclaringModule.ModuleType != ModuleType.Stub) { + var expectedDoc = expectedMemberType.Documentation?.Trim(); + var actualDoc = actualMemberType.Documentation?.Trim(); + + Debug.Assert(expectedDoc == actualDoc); + if (string.IsNullOrEmpty(expectedDoc)) { + assertion.ForCondition(string.IsNullOrEmpty(actualDoc)) + .FailWith($"Expected python type of '{GetName(actualMemberType)}.{n}' to have no documentation{{reason}}, but it has '{actualDoc}'"); + } else { + assertion.ForCondition(actualDoc.EqualsOrdinal(expectedDoc)) + .FailWith($"Expected python type of '{GetName(actualMemberType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualDoc}'"); } + } + #endregion + + #region Member type specific checks + switch (actualMemberType.MemberType) { + case PythonMemberType.Class: + // Restored collections (like instance of tuple) turn into classes + // rather than into collections with content since we don't track + // collection content in libraries. We don't compare qualified names + // since original module may be source or a stub and that is not + // preserved during restore. + // subjectMemberType.QualifiedName.Should().Be(otherMemberType.QualifiedName); + break; + case PythonMemberType.Function: + case PythonMemberType.Method: + actualMemberType.Should().BeAssignableTo(); + expectedMemberType.Should().BeAssignableTo(); + if (actualMemberType is IPythonFunctionType subjectFunction) { + var otherFunction = (IPythonFunctionType)expectedMemberType; + subjectFunction.Should().HaveSameOverloadsAs(otherFunction); + } + + break; + case PythonMemberType.Property: + actualMemberType.Should().BeAssignableTo(); + expectedMemberType.Should().BeAssignableTo(); + break; + case PythonMemberType.Unknown: + actualMemberType.IsUnknown().Should().BeTrue(); + break; + } + #endregion - break; - case PythonMemberType.Property: - actualMemberType.Should().BeAssignableTo(); - expectedMemberType.Should().BeAssignableTo(); - break; - case PythonMemberType.Unknown: - actualMemberType.IsUnknown().Should().BeTrue(); - break; + // Recurse into members. + actualMemberType.Should().HaveSameMembersAs(expectedMemberType); } } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index f8a0bb209..11db80493 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -204,7 +203,7 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { if (member == null) { var containerName = mc is IPythonType t ? t.Name : ""; - Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); + //Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); break; } diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index 5cbefbef9..43362e8b9 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -75,6 +75,9 @@ def func2() -> C2: ... [TestMethod, Priority(0)] public Task Sys() => TestModule("sys"); + [TestMethod, Priority(0)] + public Task Os() => TestModule("os"); + [TestMethod, Priority(0)] public Task Numpy() => TestModule("numpy"); From d7ca27f25c23fb217fbaea3253bc7d5149e46ae1 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 8 Nov 2019 11:34:35 -0800 Subject: [PATCH 032/102] Fix instance/class mismatch in operators eval --- .../Evaluation/ExpressionEval.Operators.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs index 15572a1a4..c0bb0b92e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs @@ -89,10 +89,13 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption if (op.IsComparison()) { return Interpreter.GetBuiltinType(BuiltinTypeId.Bool); } - return UnknownType; } + var leftValue = left is IVariable v1 ? v1.Value : left; + var rightValue = right is IVariable v2 ? v2.Value : right; + var isInstance = leftValue is IPythonInstance || rightValue is IPythonInstance; + var leftType = left.GetPythonType(); var rightType = right.GetPythonType(); @@ -121,7 +124,7 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption if (leftIsSupported && rightIsSupported) { if (TryGetValueFromBuiltinBinaryOp(op, leftTypeId, rightTypeId, Interpreter.LanguageVersion.Is3x(), out var member)) { - return member; + return isInstance ? new PythonInstance(member.GetPythonType()) : member; } } @@ -136,11 +139,10 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption ret = CallOperator(op, left, leftType, right, rightType, expr, tryLeft: false); } - if (!ret.IsUnknown()) { - return ret; + if (ret.IsUnknown()) { + ret = op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left; } - - return op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left; + return isInstance ? new PythonInstance(ret.GetPythonType()) : ret; } if (rightIsSupported) { From 79e8b2d4490c59dbc0389302da8deb976f18e256 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 8 Nov 2019 11:35:23 -0800 Subject: [PATCH 033/102] Properly handle variables in type.AddMember --- src/Analysis/Ast/Impl/Types/PythonType.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 11770248a..e7ab6c009 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -142,7 +142,7 @@ internal IMember AddMember(string name, IMember member, bool overwrite) { lock (_lock) { if (!_readonly) { if (overwrite || !Members.ContainsKey(name)) { - WritableMembers[name] = member; + WritableMembers[name] = member is IVariable v ? v.Value : member; } } return member; From f197a6dea7edbc57edf469b47b0597dae9de6c9b Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 8 Nov 2019 11:36:45 -0800 Subject: [PATCH 034/102] Fix function inner members restore --- src/Caching/Impl/Lazy/PythonLazyClassType.cs | 19 +++++++------------ .../Impl/Lazy/PythonLazyFunctionType.cs | 2 +- src/Caching/Impl/Lazy/PythonLazyType.cs | 13 +++++++++++++ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs index 3a8e8fc9f..466dc6fb0 100644 --- a/src/Caching/Impl/Lazy/PythonLazyClassType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -19,6 +19,7 @@ using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing; @@ -35,18 +36,6 @@ public PythonLazyClassType(ClassModel model, ModuleFactory mf, IGlobalScope gs, SetInnerType(_cls); } - #region IPythonType - public override IMember GetMember(string name) { - EnsureContent(); - return base.GetMember(name); - } - - public override IEnumerable GetMemberNames() { - EnsureContent(); - return base.GetMemberNames(); - } - #endregion - #region IPythonClassType public IPythonType CreateSpecificType(IArgumentSet typeArguments) { EnsureContent(); @@ -115,6 +104,7 @@ protected override void EnsureContent(ClassModel cm) { foreach (var model in allMemberModels) { _cls.AddMember(model.Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _cls), false); } + _cls.AddMember("__class__", _cls, true); } private IEnumerable CreateBases(ClassModel cm, ModuleFactory mf, IGlobalScope gs) { @@ -149,6 +139,11 @@ private IEnumerable CreateBases(ClassModel cm, ModuleFactory mf, IG Debug.Fail("Generic class does not have generic base."); } } + + if (bases.Length > 0) { + _cls.AddMember("__base__", bases[0], true); + } + _cls.AddMember("__bases__", PythonCollectionType.CreateList(DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule, bases), true); return bases; } } diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs index 6d53dfa19..352376bdc 100644 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -51,7 +51,7 @@ public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScop protected override void EnsureContent(FunctionModel fm) { var innerTypes = fm.Classes.Concat(fm.Functions).ToArray(); foreach (var model in innerTypes) { - _function.AddMember(Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _function), overwrite: true); + _function.AddMember(model.Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _function), overwrite: true); } } } diff --git a/src/Caching/Impl/Lazy/PythonLazyType.cs b/src/Caching/Impl/Lazy/PythonLazyType.cs index f248bad87..2576d19ce 100644 --- a/src/Caching/Impl/Lazy/PythonLazyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyType.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -39,6 +40,18 @@ protected PythonLazyType(TModel model, ModuleFactory mf, IGlobalScope gs, IPytho public IPythonType DeclaringType { get; } + #region IPythonType + public override IMember GetMember(string name) { + EnsureContent(); + return base.GetMember(name); + } + + public override IEnumerable GetMemberNames() { + EnsureContent(); + return base.GetMemberNames(); + } + #endregion + internal void EnsureContent() { lock (_contentLock) { if (_model != null) { From 3f9ee9aa20b91d2b95a788c84b51dbb9658ce73e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sun, 10 Nov 2019 11:26:21 -0800 Subject: [PATCH 035/102] Cache models by qualified name --- .../Test/FluentAssertions/MemberAssertions.cs | 23 +- src/Caching/Impl/Models/ModuleModel.cs | 16 +- src/Caching/Impl/ModuleFactory.cs | 4 +- src/Caching/Test/AnalysisCachingTestBase.cs | 4 +- src/json1.json | 7980 +++++++++++++++++ 5 files changed, 8011 insertions(+), 16 deletions(-) create mode 100644 src/json1.json diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 491fc0edd..784b47582 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -124,14 +124,14 @@ public AndWhichConstraint HaveMember(string return new AndWhichConstraint(this, typedMember); } - public AndConstraint HaveSameMemberNamesAs(IMember member) { + public AndConstraint HaveSameMemberNamesAs(IMember member, bool recursive = false) { member.Should().BeAssignableTo(); return HaveMembers(((IMemberContainer)member).GetMemberNames(), string.Empty); } private static readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); - public void HaveSameMembersAs(IMember expected, string because = "", params object[] becauseArgs) { + public void HaveSameMembersAs(IMember expected, bool recursive = false, string because = "", params object[] becauseArgs) { var expectedContainer = expected.Should().BeAssignableTo().Which; var actualContainer = Subject.GetPythonType(); @@ -140,14 +140,16 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje return; } var actualNames = actualContainer.GetMemberNames().ToArray(); - var expectedNames = expectedContainer.GetMemberNames().ToArray(); + var expectedNames = expectedContainer.GetMemberNames().Except(Enumerable.Repeat("", 1)).ToArray(); var errorMessage = GetAssertCollectionOnlyContainsMessage(actualNames, expectedNames, GetQuotedName(Subject), "member", "members"); var assertion = Execute.Assertion.BecauseOf(because, becauseArgs); + Debug.Assert(errorMessage == null); assertion.ForCondition(errorMessage == null).FailWith(errorMessage); - foreach (var n in actualNames.Except(Enumerable.Repeat("__base__", 1))) { + // TODO: In restored case __iter__ is a function while in analysis it is a specialized class. + foreach (var n in actualNames.Except(new[] { "__base__", "__iter__" })) { var actualMember = actualContainer.GetMember(n); var expectedMember = expectedContainer.GetMember(n); @@ -156,7 +158,7 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. if (expectedMember is IPythonInstance && !expectedMember.IsUnknown()) { - Debug.Assert(actualMember is IPythonInstance); + // Debug.Assert(actualMember is IPythonInstance); assertion.ForCondition(actualMember is IPythonInstance) .FailWith($"Expected '{GetName(actualContainer)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); } @@ -231,7 +233,16 @@ public void HaveSameMembersAs(IMember expected, string because = "", params obje #endregion // Recurse into members. - actualMemberType.Should().HaveSameMembersAs(expectedMemberType); + // https://github.com/microsoft/python-language-server/issues/1533 + // Ex 'BigEndianStructure' in ctypes has attached members from stub. + // However, when running test fetching it from 'ctypes._endian' yields + // class without stub members. This affects tests with partial restoration. + + // Also, there are issues when restored object methods are + // not specialized like __iter__ or __getattribute__. + if (recursive) { + actualMemberType.Should().HaveSameMembersAs(expectedMemberType, recursive); + } } } } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 09597d577..6307b0358 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -51,6 +51,7 @@ internal sealed class ModuleModel : MemberModel { public int FileSize { get; set; } [NonSerialized] private Dictionary _modelCache; + [NonSerialized] private object _modelCacheLock = new object(); /// /// Constructs module persistent model from analysis. @@ -158,14 +159,17 @@ private static FunctionModel GetFunctionModel(IDocumentAnalysis analysis, IVaria } public override MemberModel GetModel(string name) { - if (_modelCache == null) { - _modelCache = new Dictionary(); - foreach (var m in GetMemberModels()) { - Debug.Assert(!_modelCache.ContainsKey(m.Name)); - _modelCache[m.Name] = m; + lock (_modelCacheLock) { + if (_modelCache == null) { + _modelCache = new Dictionary(); + foreach (var m in GetMemberModels()) { + Debug.Assert(!_modelCache.ContainsKey(m.QualifiedName)); + _modelCache[m.QualifiedName] = m; + } } + + return _modelCache.TryGetValue(name, out var model) ? model : null; } - return _modelCache.TryGetValue(name, out var model) ? model : null; } protected override IEnumerable GetMemberModels() diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 11db80493..a4e138fe6 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -103,8 +103,8 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { } var nextModel = currentModel.GetModel(memberName); - Debug.Assert(nextModel != null, - $"Unable to find {string.Join(".", memberNames)} in module {Module.Name}"); + //Debug.Assert(nextModel != null, + // $"Unable to find {string.Join(".", memberNames)} in module {Module.Name}"); if (nextModel == null) { return null; } diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 58bcf518e..9ebca86f6 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -62,12 +62,12 @@ internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { return dbModule; } - internal async Task CompareRestoreAsync(ModuleModel model, IPythonModule m) { + internal async Task CompareRestoreAsync(ModuleModel model, IPythonModule m, bool recursive = false) { var analyzer = Services.GetService(); await analyzer.WaitForCompleteAnalysisAsync(); using (var dbModule = CreateDbModule(model, m.FilePath)) { - dbModule.Should().HaveSameMembersAs(m); + dbModule.Should().HaveSameMembersAs(m, recursive); } } diff --git a/src/json1.json b/src/json1.json new file mode 100644 index 000000000..91e0808c8 --- /dev/null +++ b/src/json1.json @@ -0,0 +1,7980 @@ +{ + "UniqueId": "ctypes(3.7)", + "FilePath": "c:\\users\\mikhaila\\appdata\\local\\programs\\python\\python37\\lib\\ctypes\\__init__.py", + "Documentation": "create and manipulate C data types in Python", + "Functions": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "init_or_size", + "Type": "t:typing:Union[int, bytes]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "size", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):Array[ctypes(stub):c_char]$ctypes(3.7)", + "Documentation": "create_string_buffer(aBytes) -> character array\n create_string_buffer(anInteger) -> character array\n create_string_buffer(aBytes, anInteger) -> character array\n " + } + ], + "Documentation": "create_string_buffer(aBytes) -> character array\n create_string_buffer(anInteger) -> character array\n create_string_buffer(aBytes, anInteger) -> character array\n ", + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -709606206, + "Name": "create_string_buffer", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:create_string_buffer", + "IndexSpan": { + "Start": 1304, + "Length": 20 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "restype", + "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "argtypes", + "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", + "DefaultValue": null, + "Kind": 1 + }, + { + "Name": "use_errno", + "Type": "t:bool", + "DefaultValue": "i:...", + "Kind": 3 + }, + { + "Name": "use_last_error", + "Type": "t:bool", + "DefaultValue": "i:...", + "Kind": 3 + } + ], + "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Documentation": "CFUNCTYPE(restype, *argtypes,\n use_errno=False, use_last_error=False) -> function prototype.\n\n restype: the result type\n argtypes: a sequence specifying the argument types\n\n The function prototype can be called in different ways to create a\n callable object:\n\n prototype(integer address) -> foreign function\n prototype(callable) -> create and return a C callable function from callable\n prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method\n prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal\n prototype((function name, dll object)[, paramflags]) -> foreign function exported by name\n " + } + ], + "Documentation": "CFUNCTYPE(restype, *argtypes,\n use_errno=False, use_last_error=False) -> function prototype.\n\n restype: the result type\n argtypes: a sequence specifying the argument types\n\n The function prototype can be called in different ways to create a\n callable object:\n\n prototype(integer address) -> foreign function\n prototype(callable) -> create and return a C callable function from callable\n prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method\n prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal\n prototype((function name, dll object)[, paramflags]) -> foreign function exported by name\n ", + "Attributes": 0, + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": -1921867084, + "Name": "_argtypes_", + "DeclaringModuleId": null, + "QualifiedName": "_argtypes_", + "IndexSpan": null + }, + { + "Value": null, + "Id": -470655251, + "Name": "_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_restype_", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1428625440, + "Name": "_flags_", + "DeclaringModuleId": null, + "QualifiedName": "_flags_", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1267928994, + "Name": "CFunctionType", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:CFUNCTYPE.CFunctionType", + "IndexSpan": { + "Start": 3342, + "Length": 13 + } + } + ], + "Functions": [], + "Id": 1098372874, + "Name": "CFUNCTYPE", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:CFUNCTYPE", + "IndexSpan": { + "Start": 2184, + "Length": 9 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "restype", + "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "argtypes", + "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", + "DefaultValue": null, + "Kind": 1 + }, + { + "Name": "use_errno", + "Type": "t:bool", + "DefaultValue": "i:...", + "Kind": 3 + }, + { + "Name": "use_last_error", + "Type": "t:bool", + "DefaultValue": "i:...", + "Kind": 3 + } + ], + "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": -1921867084, + "Name": "_argtypes_", + "DeclaringModuleId": null, + "QualifiedName": "_argtypes_", + "IndexSpan": null + }, + { + "Value": null, + "Id": -470655251, + "Name": "_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_restype_", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1428625440, + "Name": "_flags_", + "DeclaringModuleId": null, + "QualifiedName": "_flags_", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -349853673, + "Name": "WinFunctionType", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:WINFUNCTYPE.WinFunctionType", + "IndexSpan": { + "Start": 4278, + "Length": 15 + } + } + ], + "Functions": [], + "Id": -1586540733, + "Name": "WINFUNCTYPE", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:WINFUNCTYPE", + "IndexSpan": { + "Start": 3742, + "Length": 11 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "typ", + "Type": null, + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "typecode", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -756102656, + "Name": "_check_size", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:_check_size", + "IndexSpan": { + "Start": 4866, + "Length": 11 + } + }, + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 774351978, + "Name": "_reset_cache", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:_reset_cache", + "IndexSpan": { + "Start": 8078, + "Length": 12 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "init_or_size", + "Type": "t:typing:Union[int, Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "size", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):Array[ctypes(stub):c_wchar]$ctypes(3.7)", + "Documentation": "create_unicode_buffer(aString) -> character array\n create_unicode_buffer(anInteger) -> character array\n create_unicode_buffer(aString, anInteger) -> character array\n " + } + ], + "Documentation": "create_unicode_buffer(aString) -> character array\n create_unicode_buffer(anInteger) -> character array\n create_unicode_buffer(aString, anInteger) -> character array\n ", + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 861572558, + "Name": "create_unicode_buffer", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:create_unicode_buffer", + "IndexSpan": { + "Start": 8792, + "Length": 21 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "pointer", + "Type": null, + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "cls", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1963781452, + "Name": "SetPointerType", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:SetPointerType", + "IndexSpan": { + "Start": 9382, + "Length": 14 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "typ", + "Type": null, + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "len", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 721023106, + "Name": "ARRAY", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:ARRAY", + "IndexSpan": { + "Start": 9765, + "Length": 5 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "code", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + }, + { + "Name": "desc", + "Type": "t:str", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:OSError", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 298941635, + "Name": "WinError", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:WinError", + "IndexSpan": { + "Start": 14822, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "restype", + "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "argtypes", + "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", + "DefaultValue": null, + "Kind": 1 + } + ], + "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [ + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": -1921867084, + "Name": "_argtypes_", + "DeclaringModuleId": null, + "QualifiedName": "_argtypes_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": -470655251, + "Name": "_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_restype_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1267928994, + "Name": "CFunctionType", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:PYFUNCTYPE.CFunctionType", + "IndexSpan": { + "Start": 15704, + "Length": 13 + } + } + ], + "Functions": [], + "Id": -1002463138, + "Name": "PYFUNCTYPE", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:PYFUNCTYPE", + "IndexSpan": { + "Start": 15661, + "Length": 10 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "obj", + "Type": "t:typing:Union[ctypes(stub):_CData, ctypes(stub):_CArgObject]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "type", + "Type": "t:typing:Type[ctypes:_PT]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes:_PT$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 24287190, + "Name": "cast", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:cast", + "IndexSpan": { + "Start": 15955, + "Length": 4 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "address", + "Type": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "size", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:bytes", + "Documentation": "string_at(addr[, size]) -> string\n\n Return the string at addr." + } + ], + "Documentation": "string_at(addr[, size]) -> string\n\n Return the string at addr.", + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -414124822, + "Name": "string_at", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:string_at", + "IndexSpan": { + "Start": 16081, + "Length": 9 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "address", + "Type": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "size", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:str", + "Documentation": "wstring_at(addr[, size]) -> string\n\n Return the string at addr." + } + ], + "Documentation": "wstring_at(addr[, size]) -> string\n\n Return the string at addr.", + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -479760415, + "Name": "wstring_at", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:wstring_at", + "IndexSpan": { + "Start": 16392, + "Length": 10 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "rclsid", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "riid", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "ppv", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1849292722, + "Name": "DllGetClassObject", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:DllGetClassObject", + "IndexSpan": { + "Start": 16592, + "Length": 17 + } + }, + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1652175700, + "Name": "DllCanUnloadNow", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:DllCanUnloadNow", + "IndexSpan": { + "Start": 16912, + "Length": 15 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "code", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:str", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1776340922, + "Name": "FormatError", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):FormatError", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "obj_or_type", + "Type": "t:typing:Union[ctypes(stub):_CData, typing:Type[ctypes(stub):_CData]]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1964161937, + "Name": "sizeof", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):sizeof", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "obj", + "Type": "t:ctypes(stub):_CData$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "offset", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):_CArgObject$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 752693029, + "Name": "byref", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):byref", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "obj", + "Type": "t:ctypes(stub):_CData$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 649701012, + "Name": "addressof", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):addressof", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "obj_or_type", + "Type": "t:typing:Union[ctypes(stub):_CData, typing:Type[ctypes(stub):_CData]]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1543031724, + "Name": "alignment", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):alignment", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "obj", + "Type": "t:ctypes(stub):_CData$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "size", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1996689525, + "Name": "resize", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):resize", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 908891526, + "Name": "get_errno", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):get_errno", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "value", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 694274450, + "Name": "set_errno", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):set_errno", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "type", + "Type": "t:typing:Type[ctypes:_CT]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1747309094, + "Name": "POINTER", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):POINTER", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1833420806, + "Name": "_check_HRESULT", + "DeclaringModuleId": "_ctypes(3.7)", + "QualifiedName": "_ctypes:_check_HRESULT", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -309340141, + "Name": "GetLastError", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):GetLastError", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1363574655, + "Name": "get_last_error", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):get_last_error", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "value", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -408161549, + "Name": "set_last_error", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):set_last_error", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "dst", + "Type": "t:typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "src", + "Type": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "count", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1915816145, + "Name": "memmove", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):memmove", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "dst", + "Type": "t:typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "c", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "count", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -2140005052, + "Name": "memset", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):memset", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Variables": [ + { + "Value": "t:ctypes:create_string_buffer$ctypes(3.7)", + "Id": -846863181, + "Name": "c_buffer", + "DeclaringModuleId": null, + "QualifiedName": "c_buffer", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:ctypes:c_void_p$ctypes(3.7)", + "Id": 1930638985, + "Name": "c_voidp", + "DeclaringModuleId": null, + "QualifiedName": "c_voidp", + "IndexSpan": { + "Start": 7668, + "Length": 7 + } + }, + { + "Value": "p:os$os(3.7)", + "Id": 780044, + "Name": "_os", + "DeclaringModuleId": null, + "QualifiedName": "_os", + "IndexSpan": { + "Start": 67, + "Length": 3 + } + }, + { + "Value": "p:sys$sys(3.7)", + "Id": 24185509, + "Name": "_sys", + "DeclaringModuleId": null, + "QualifiedName": "_sys", + "IndexSpan": { + "Start": 79, + "Length": 4 + } + }, + { + "Value": "i:str", + "Id": 1161199201, + "Name": "__version__", + "DeclaringModuleId": null, + "QualifiedName": "__version__", + "IndexSpan": { + "Start": 87, + "Length": 11 + } + }, + { + "Value": "t:_ctypes:_Pointer$_ctypes(3.7)", + "Id": 705525237, + "Name": "_Pointer", + "DeclaringModuleId": null, + "QualifiedName": "_Pointer", + "IndexSpan": { + "Start": 177, + "Length": 8 + } + }, + { + "Value": null, + "Id": 1824826095, + "Name": "_CFuncPtr", + "DeclaringModuleId": null, + "QualifiedName": "_CFuncPtr", + "IndexSpan": { + "Start": 219, + "Length": 9 + } + }, + { + "Value": "i:str", + "Id": 1827465399, + "Name": "_ctypes_version", + "DeclaringModuleId": null, + "QualifiedName": "_ctypes_version", + "IndexSpan": { + "Start": 265, + "Length": 15 + } + }, + { + "Value": "i:int", + "Id": -789048995, + "Name": "RTLD_LOCAL", + "DeclaringModuleId": null, + "QualifiedName": "RTLD_LOCAL", + "IndexSpan": { + "Start": 302, + "Length": 10 + } + }, + { + "Value": "i:int", + "Id": 1163726801, + "Name": "RTLD_GLOBAL", + "DeclaringModuleId": null, + "QualifiedName": "RTLD_GLOBAL", + "IndexSpan": { + "Start": 314, + "Length": 11 + } + }, + { + "Value": "t:struct(stub):calcsize$struct(3.7)", + "Id": -314342658, + "Name": "_calcsize", + "DeclaringModuleId": null, + "QualifiedName": "_calcsize", + "IndexSpan": { + "Start": 395, + "Length": 9 + } + }, + { + "Value": "i:int", + "Id": -756921704, + "Name": "DEFAULT_MODE", + "DeclaringModuleId": null, + "QualifiedName": "DEFAULT_MODE", + "IndexSpan": { + "Start": 585, + "Length": 12 + } + }, + { + "Value": "i:int", + "Id": 1661077446, + "Name": "_FUNCFLAG_CDECL", + "DeclaringModuleId": null, + "QualifiedName": "_FUNCFLAG_CDECL", + "IndexSpan": { + "Start": 981, + "Length": 15 + } + }, + { + "Value": "i:int", + "Id": -1361874057, + "Name": "_FUNCFLAG_PYTHONAPI", + "DeclaringModuleId": null, + "QualifiedName": "_FUNCFLAG_PYTHONAPI", + "IndexSpan": { + "Start": 1028, + "Length": 19 + } + }, + { + "Value": "i:int", + "Id": 840545479, + "Name": "_FUNCFLAG_USE_ERRNO", + "DeclaringModuleId": null, + "QualifiedName": "_FUNCFLAG_USE_ERRNO", + "IndexSpan": { + "Start": 1079, + "Length": 19 + } + }, + { + "Value": "i:int", + "Id": -675549293, + "Name": "_FUNCFLAG_USE_LASTERROR", + "DeclaringModuleId": null, + "QualifiedName": "_FUNCFLAG_USE_LASTERROR", + "IndexSpan": { + "Start": 1134, + "Length": 23 + } + }, + { + "Value": "i:dict$ctypes(3.7)", + "Id": -843695483, + "Name": "_c_functype_cache", + "DeclaringModuleId": null, + "QualifiedName": "_c_functype_cache", + "IndexSpan": { + "Start": 2156, + "Length": 17 + } + }, + { + "Value": "t:_ctypes:LoadLibrary$_ctypes(3.7)", + "Id": -1485458438, + "Name": "_dlopen", + "DeclaringModuleId": null, + "QualifiedName": "_dlopen", + "IndexSpan": { + "Start": 3630, + "Length": 7 + } + }, + { + "Value": "i:int", + "Id": 339883258, + "Name": "_FUNCFLAG_STDCALL", + "DeclaringModuleId": null, + "QualifiedName": "_FUNCFLAG_STDCALL", + "IndexSpan": { + "Start": 3683, + "Length": 17 + } + }, + { + "Value": "i:dict$ctypes(3.7)", + "Id": 2115421164, + "Name": "_win_functype_cache", + "DeclaringModuleId": null, + "QualifiedName": "_win_functype_cache", + "IndexSpan": { + "Start": 3708, + "Length": 19 + } + }, + { + "Value": "i:dict", + "Id": 456857255, + "Name": "_pointer_type_cache", + "DeclaringModuleId": null, + "QualifiedName": "_pointer_type_cache", + "IndexSpan": { + "Start": 7837, + "Length": 19 + } + }, + { + "Value": "i:ctypes:LibraryLoader[ctypes(stub):CDLL]$ctypes(3.7)", + "Id": 24289848, + "Name": "cdll", + "DeclaringModuleId": null, + "QualifiedName": "cdll", + "IndexSpan": { + "Start": 14328, + "Length": 4 + } + }, + { + "Value": "i:ctypes:LibraryLoader[ctypes(stub):PyDLL]$ctypes(3.7)", + "Id": 765609092, + "Name": "pydll", + "DeclaringModuleId": null, + "QualifiedName": "pydll", + "IndexSpan": { + "Start": 14356, + "Length": 5 + } + }, + { + "Value": "i:ctypes:PyDLL$ctypes(3.7)", + "Id": -1710066297, + "Name": "pythonapi", + "DeclaringModuleId": null, + "QualifiedName": "pythonapi", + "IndexSpan": { + "Start": 14414, + "Length": 9 + } + }, + { + "Value": "i:ctypes:LibraryLoader[ctypes(stub):WinDLL]$ctypes(3.7)", + "Id": -1850003873, + "Name": "windll", + "DeclaringModuleId": null, + "QualifiedName": "windll", + "IndexSpan": { + "Start": 14637, + "Length": 6 + } + }, + { + "Value": "i:ctypes:LibraryLoader[ctypes(stub):OleDLL]$ctypes(3.7)", + "Id": -2076534637, + "Name": "oledll", + "DeclaringModuleId": null, + "QualifiedName": "oledll", + "IndexSpan": { + "Start": 14673, + "Length": 6 + } + }, + { + "Value": "i:int", + "Id": 357987474, + "Name": "_memmove_addr", + "DeclaringModuleId": null, + "QualifiedName": "_memmove_addr", + "IndexSpan": { + "Start": 15359, + "Length": 13 + } + }, + { + "Value": "i:int", + "Id": -2071594405, + "Name": "_memset_addr", + "DeclaringModuleId": null, + "QualifiedName": "_memset_addr", + "IndexSpan": { + "Start": 15374, + "Length": 12 + } + }, + { + "Value": "i:int", + "Id": -1481723017, + "Name": "_string_at_addr", + "DeclaringModuleId": null, + "QualifiedName": "_string_at_addr", + "IndexSpan": { + "Start": 15388, + "Length": 15 + } + }, + { + "Value": "i:int", + "Id": 1230836425, + "Name": "_cast_addr", + "DeclaringModuleId": null, + "QualifiedName": "_cast_addr", + "IndexSpan": { + "Start": 15405, + "Length": 10 + } + }, + { + "Value": "i:typing:Any$typing(3.7)", + "Id": 749251175, + "Name": "_cast", + "DeclaringModuleId": null, + "QualifiedName": "_cast", + "IndexSpan": { + "Start": 15876, + "Length": 5 + } + }, + { + "Value": "i:typing:Any$typing(3.7)", + "Id": -58403591, + "Name": "_string_at", + "DeclaringModuleId": null, + "QualifiedName": "_string_at", + "IndexSpan": { + "Start": 16007, + "Length": 10 + } + }, + { + "Value": "i:int", + "Id": -997588386, + "Name": "_wstring_at_addr", + "DeclaringModuleId": null, + "QualifiedName": "_wstring_at_addr", + "IndexSpan": { + "Start": 16252, + "Length": 16 + } + }, + { + "Value": "i:typing:Any$typing(3.7)", + "Id": 1957663154, + "Name": "_wstring_at", + "DeclaringModuleId": null, + "QualifiedName": "_wstring_at", + "IndexSpan": { + "Start": 16312, + "Length": 11 + } + }, + { + "Value": "t:ctypes._endian:BigEndianStructure$ctypes._endian(3.7)", + "Id": 2047389551, + "Name": "BigEndianStructure", + "DeclaringModuleId": null, + "QualifiedName": "BigEndianStructure", + "IndexSpan": { + "Start": 17163, + "Length": 18 + } + }, + { + "Value": "t:ctypes:c_ushort$ctypes(3.7)", + "Id": 24533035, + "Name": "kind", + "DeclaringModuleId": null, + "QualifiedName": "kind", + "IndexSpan": { + "Start": 17284, + "Length": 4 + } + }, + { + "Value": "t:typing:Union[typing:Tuple[int], typing:Tuple[int, str], typing:Tuple[int, str, typing:Any]]$typing(3.7)", + "Id": 779038, + "Name": "_PF", + "DeclaringModuleId": null, + "QualifiedName": "_PF", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int]$typing(3.7)", + "Id": 474673944, + "Name": "_CVoidPLike", + "DeclaringModuleId": null, + "QualifiedName": "_CVoidPLike", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", + "Id": 2128974387, + "Name": "_CVoidConstPLike", + "DeclaringModuleId": null, + "QualifiedName": "_CVoidConstPLike", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Classes": [ + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[ctypes:_T]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:ctypes:_T$ctypes(3.7)", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1315475454, + "Name": "py_object", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:py_object", + "IndexSpan": { + "Start": 5392, + "Length": 9 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1927666089, + "Name": "c_short", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_short", + "IndexSpan": { + "Start": 5634, + "Length": 7 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -304687718, + "Name": "c_ushort", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_ushort", + "IndexSpan": { + "Start": 5706, + "Length": 8 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1863096239, + "Name": "c_long", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_long", + "IndexSpan": { + "Start": 5780, + "Length": 6 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1929632158, + "Name": "c_ulong", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_ulong", + "IndexSpan": { + "Start": 5850, + "Length": 7 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 752833628, + "Name": "c_int", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_int", + "IndexSpan": { + "Start": 6087, + "Length": 5 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1863358605, + "Name": "c_uint", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_uint", + "IndexSpan": { + "Start": 6167, + "Length": 6 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[float]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:float", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1915778953, + "Name": "c_float", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_float", + "IndexSpan": { + "Start": 6245, + "Length": 7 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[float]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:float", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -794702780, + "Name": "c_double", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_double", + "IndexSpan": { + "Start": 6317, + "Length": 8 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[float]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:float", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1655090112, + "Name": "c_longdouble", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_longdouble", + "IndexSpan": { + "Start": 6317, + "Length": 8 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1656614571, + "Name": "c_longlong", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_longlong", + "IndexSpan": { + "Start": 6707, + "Length": 10 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 877961498, + "Name": "c_ulonglong", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_ulonglong", + "IndexSpan": { + "Start": 6797, + "Length": 11 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1929344042, + "Name": "c_ubyte", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_ubyte", + "IndexSpan": { + "Start": 7015, + "Length": 7 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1862808123, + "Name": "c_byte", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_byte", + "IndexSpan": { + "Start": 7190, + "Length": 6 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[bytes]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:c_char$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "value", + "Type": "t:typing:Union[int, bytes]$typing(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):c_char.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:bytes", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1862821001, + "Name": "c_char", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_char", + "IndexSpan": { + "Start": 7312, + "Length": 6 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_PointerLike$ctypes(3.7)", + "t:ctypes(stub):_SimpleCData[bytes]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:c_char_p$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "value", + "Type": "t:typing:Union[int, bytes]$typing(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):c_char_p.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + }, + { + "Value": "i:bytes", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -830377414, + "Name": "c_char_p", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_char_p", + "IndexSpan": { + "Start": 7434, + "Length": 8 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_PointerLike$ctypes(3.7)", + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -279734024, + "Name": "c_void_p", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_void_p", + "IndexSpan": { + "Start": 7625, + "Length": 8 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[bool]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:c_bool$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "value", + "Type": "t:bool", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):c_bool.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:bool", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1862798365, + "Name": "c_bool", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_bool", + "IndexSpan": { + "Start": 7756, + "Length": 6 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_PointerLike$ctypes(3.7)", + "t:ctypes(stub):_SimpleCData[ctypes:_T]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:c_wchar_p$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "value", + "Type": "t:typing:Union[int, Unknown]$typing(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):c_wchar_p.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + }, + { + "Value": "i:ctypes:_T$ctypes(3.7)", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 461138667, + "Name": "c_wchar_p", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_wchar_p", + "IndexSpan": { + "Start": 7866, + "Length": 9 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[ctypes:_T]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:ctypes:_T$ctypes(3.7)", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1931203962, + "Name": "c_wchar", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_wchar", + "IndexSpan": { + "Start": 8030, + "Length": 7 + } + }, + { + "Documentation": "An instance of this class represents a loaded dll/shared\n library, exporting functions using the standard C calling\n convention (named 'cdecl' on Windows).\n\n The exported functions can be accessed as attributes, or by\n indexing with the function name. Examples:\n\n .qsort -> callable object\n ['qsort'] -> callable object\n\n Calling the functions releases the Python GIL during the call and\n reacquires it afterwards.\n ", + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:CDLL$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "mode", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + }, + { + "Name": "handle", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + }, + { + "Name": "use_errno", + "Type": "t:bool", + "DefaultValue": "i:...", + "Kind": 0 + }, + { + "Name": "use_last_error", + "Type": "t:bool", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": "An instance of this class represents a loaded dll/shared\n library, exporting functions using the standard C calling\n convention (named 'cdecl' on Windows).\n\n The exported functions can be accessed as attributes, or by\n indexing with the function name. Examples:\n\n .qsort -> callable object\n ['qsort'] -> callable object\n\n Calling the functions releases the Python GIL during the call and\n reacquires it afterwards.\n " + } + ], + "Documentation": "An instance of this class represents a loaded dll/shared\n library, exporting functions using the standard C calling\n convention (named 'cdecl' on Windows).\n\n The exported functions can be accessed as attributes, or by\n indexing with the function name. Examples:\n\n .qsort -> callable object\n ['qsort'] -> callable object\n\n Calling the functions releases the Python GIL during the call and\n reacquires it afterwards.\n ", + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:CDLL.__init__", + "IndexSpan": { + "Start": 10550, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:CDLL$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -521760752, + "Name": "__getattr__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:CDLL.__getattr__", + "IndexSpan": { + "Start": 11882, + "Length": 11 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:CDLL$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -293179214, + "Name": "__getitem__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:CDLL.__getitem__", + "IndexSpan": { + "Start": 12112, + "Length": 11 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": 2021379179, + "Name": "_func_flags_", + "DeclaringModuleId": null, + "QualifiedName": "_func_flags_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": -730968904, + "Name": "_func_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_func_restype_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 749578675, + "Name": "_name", + "DeclaringModuleId": null, + "QualifiedName": "_name", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1381141680, + "Name": "_handle", + "DeclaringModuleId": null, + "QualifiedName": "_handle", + "IndexSpan": null + }, + { + "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Id": 596453698, + "Name": "_FuncPtr", + "DeclaringModuleId": null, + "QualifiedName": "_FuncPtr", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 23304760, + "Name": "CDLL", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:CDLL", + "IndexSpan": { + "Start": 9883, + "Length": 4 + } + }, + { + "Documentation": "This class represents the Python library itself. It allows\n accessing Python API functions. The GIL is not released, and\n Python exceptions are handled correctly.\n ", + "Bases": [ + "t:ctypes:CDLL$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": 2021379179, + "Name": "_func_flags_", + "DeclaringModuleId": null, + "QualifiedName": "_func_flags_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": -730968904, + "Name": "_func_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_func_restype_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 749578675, + "Name": "_name", + "DeclaringModuleId": null, + "QualifiedName": "_name", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1381141680, + "Name": "_handle", + "DeclaringModuleId": null, + "QualifiedName": "_handle", + "IndexSpan": null + }, + { + "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Id": 596453698, + "Name": "_FuncPtr", + "DeclaringModuleId": null, + "QualifiedName": "_FuncPtr", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 736024644, + "Name": "PyDLL", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:PyDLL", + "IndexSpan": { + "Start": 12328, + "Length": 5 + } + }, + { + "Documentation": "This class represents a dll exporting functions using the\n Windows stdcall calling convention.\n ", + "Bases": [ + "t:ctypes:CDLL$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": 2021379179, + "Name": "_func_flags_", + "DeclaringModuleId": null, + "QualifiedName": "_func_flags_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": -730968904, + "Name": "_func_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_func_restype_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 749578675, + "Name": "_name", + "DeclaringModuleId": null, + "QualifiedName": "_name", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1381141680, + "Name": "_handle", + "DeclaringModuleId": null, + "QualifiedName": "_handle", + "IndexSpan": null + }, + { + "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Id": 596453698, + "Name": "_FuncPtr", + "DeclaringModuleId": null, + "QualifiedName": "_FuncPtr", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1528798815, + "Name": "WinDLL", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:WinDLL", + "IndexSpan": { + "Start": 12626, + "Length": 6 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -975409554, + "Name": "HRESULT", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:HRESULT", + "IndexSpan": { + "Start": 12975, + "Length": 7 + } + }, + { + "Documentation": "This class represents a dll exporting functions using the\n Windows stdcall calling convention, and returning HRESULT.\n HRESULT error values are automatically raised as OSError\n exceptions.\n ", + "Bases": [ + "t:ctypes:CDLL$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": 2021379179, + "Name": "_func_flags_", + "DeclaringModuleId": null, + "QualifiedName": "_func_flags_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": -730968904, + "Name": "_func_restype_", + "DeclaringModuleId": null, + "QualifiedName": "_func_restype_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:str", + "Id": 749578675, + "Name": "_name", + "DeclaringModuleId": null, + "QualifiedName": "_name", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1381141680, + "Name": "_handle", + "DeclaringModuleId": null, + "QualifiedName": "_handle", + "IndexSpan": null + }, + { + "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", + "Id": 596453698, + "Name": "_FuncPtr", + "DeclaringModuleId": null, + "QualifiedName": "_FuncPtr", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1302268051, + "Name": "OleDLL", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:OleDLL", + "IndexSpan": { + "Start": 13561, + "Length": 6 + } + }, + { + "Documentation": null, + "Bases": [ + "t:typing:Generic$typing(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "dlltype", + "Type": "t:typing:Type[ctypes:_DLLT]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:LibraryLoader.__init__", + "IndexSpan": { + "Start": 13930, + "Length": 8 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes:_DLLT$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -521760752, + "Name": "__getattr__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:LibraryLoader.__getattr__", + "IndexSpan": { + "Start": 13999, + "Length": 11 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes:_DLLT$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -293179214, + "Name": "__getitem__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:LibraryLoader.__getitem__", + "IndexSpan": { + "Start": 14192, + "Length": 11 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes:_DLLT$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -4764578, + "Name": "LoadLibrary", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:LibraryLoader.LoadLibrary", + "IndexSpan": { + "Start": 14264, + "Length": 11 + } + } + ], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [ + "_DLLT" + ], + "GenericParameterValues": [], + "Id": 49337815, + "Name": "LibraryLoader", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:LibraryLoader", + "IndexSpan": { + "Start": 13898, + "Length": 13 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1863006044, + "Name": "c_int8", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_int8", + "IndexSpan": { + "Start": 7190, + "Length": 6 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1929541963, + "Name": "c_uint8", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_uint8", + "IndexSpan": { + "Start": 7015, + "Length": 7 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 740351224, + "Name": "Union", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Union", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1923643164, + "Name": "Structure", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Structure", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:typing:Generic$typing(3.7)", + "t:ctypes(stub):_CData$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "args", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 1 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Array.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "i", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:typing:Any$typing(3.7)", + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "s", + "Type": "t:slice", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:typing:List[typing:Any]$typing(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -293179214, + "Name": "__getitem__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Array.__getitem__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "i", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "o", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "s", + "Type": "t:slice", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "o", + "Type": "t:typing:Iterable[typing:Any]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -507796290, + "Name": "__setitem__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Array.__setitem__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:list_iterator", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 971292143, + "Name": "__iter__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Array.__iter__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):Array$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:int", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1628904226, + "Name": "__len__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Array.__len__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": -496057839, + "Name": "_length_", + "DeclaringModuleId": null, + "QualifiedName": "_length_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": 1768361533, + "Name": "_type_", + "DeclaringModuleId": null, + "QualifiedName": "_type_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:bytes", + "Id": 797873, + "Name": "raw", + "DeclaringModuleId": null, + "QualifiedName": "raw", + "IndexSpan": null + }, + { + "Value": "i:bytes", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [ + "_T" + ], + "GenericParameterValues": [], + "Id": 722008194, + "Name": "Array", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):Array", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": "Common base class for all non-exit exceptions.", + "Bases": [ + "t:Exception", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1992130164, + "Name": "ArgumentError", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):ArgumentError", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:typing:Generic$typing(3.7)", + "t:ctypes(stub):_CData$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_SimpleCData$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "value", + "Type": "t:ctypes:_T$ctypes(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_SimpleCData.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:ctypes:_T$ctypes(3.7)", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [ + "_T" + ], + "GenericParameterValues": [], + "Id": 1441346515, + "Name": "_SimpleCData", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_SimpleCData", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_PointerLike$ctypes(3.7)", + "t:ctypes(stub):_CData$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):pointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "arg", + "Type": "t:ellipsis", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):pointer.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):pointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "i", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):pointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "s", + "Type": "t:slice", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:typing:List[Unknown]$typing(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -293179214, + "Name": "__getitem__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):pointer.__getitem__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):pointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "i", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "o", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):pointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "s", + "Type": "t:slice", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "o", + "Type": "t:typing:Iterable[Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -507796290, + "Name": "__setitem__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):pointer.__setitem__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": 1768361533, + "Name": "_type_", + "DeclaringModuleId": null, + "QualifiedName": "_type_", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": null, + "Id": 810898481, + "Name": "contents", + "DeclaringModuleId": null, + "QualifiedName": "contents", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1029326406, + "Name": "pointer", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):pointer", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -370655191, + "Name": "c_size_t", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_size_t", + "IndexSpan": { + "Start": 6797, + "Length": 11 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1665813462, + "Name": "c_ssize_t", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_ssize_t", + "IndexSpan": { + "Start": 6707, + "Length": 10 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):Structure$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1726755221, + "Name": "LittleEndianStructure", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):LittleEndianStructure", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1918612353, + "Name": "c_int16", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_int16", + "IndexSpan": { + "Start": 5634, + "Length": 7 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1918612411, + "Name": "c_int32", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_int32", + "IndexSpan": { + "Start": 5634, + "Length": 7 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1918612506, + "Name": "c_int64", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_int64", + "IndexSpan": { + "Start": 5634, + "Length": 7 + } + }, + { + "Documentation": "XXX to be provided", + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -313741454, + "Name": "c_uint16", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_uint16", + "IndexSpan": { + "Start": 5706, + "Length": 8 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -313741396, + "Name": "c_uint32", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_uint32", + "IndexSpan": { + "Start": 5706, + "Length": 8 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": 770443194, + "Name": "value", + "DeclaringModuleId": null, + "QualifiedName": "value", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -313741301, + "Name": "c_uint64", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes:c_uint64", + "IndexSpan": { + "Start": 5706, + "Length": 8 + } + }, + { + "Documentation": "type(object_or_name, bases, dict)\ntype(object) -> the object's type\ntype(name, bases, dict) -> a new type", + "Bases": [ + "t:type", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[ctypes:_CT]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "other", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):Array[ctypes:_CT]$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -1627505971, + "Name": "__mul__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CDataMeta.__mul__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[ctypes:_CT]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "other", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):Array[ctypes:_CT]$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 1222960745, + "Name": "__rmul__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CDataMeta.__rmul__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": null, + "Id": -1388753224, + "Name": "__basicsize__", + "DeclaringModuleId": null, + "QualifiedName": "__basicsize__", + "IndexSpan": null + }, + { + "Value": null, + "Id": -198224608, + "Name": "__dictoffset__", + "DeclaringModuleId": null, + "QualifiedName": "__dictoffset__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 1444705936, + "Name": "__flags__", + "DeclaringModuleId": null, + "QualifiedName": "__flags__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 919460331, + "Name": "__itemsize__", + "DeclaringModuleId": null, + "QualifiedName": "__itemsize__", + "IndexSpan": null + }, + { + "Value": "i:list$ctypes(3.7)", + "Id": -1627592461, + "Name": "__mro__", + "DeclaringModuleId": null, + "QualifiedName": "__mro__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 1097116834, + "Name": "__name__", + "DeclaringModuleId": null, + "QualifiedName": "__name__", + "IndexSpan": null + }, + { + "Value": null, + "Id": -1879833807, + "Name": "__qualname__", + "DeclaringModuleId": null, + "QualifiedName": "__qualname__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 174109373, + "Name": "__text_signature__", + "DeclaringModuleId": null, + "QualifiedName": "__text_signature__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 1521523639, + "Name": "__weakrefoffset__", + "DeclaringModuleId": null, + "QualifiedName": "__weakrefoffset__", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1144000522, + "Name": "_CDataMeta", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CDataMeta", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "source", + "Type": "t:bytearray", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "offset", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 3, + "Classes": [], + "Functions": [], + "Id": -1600044546, + "Name": "from_buffer", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CData.from_buffer", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "source", + "Type": "t:bytearray", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "offset", + "Type": "t:int", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 3, + "Classes": [], + "Functions": [], + "Id": 1037546774, + "Name": "from_buffer_copy", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CData.from_buffer_copy", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "address", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 3, + "Classes": [], + "Functions": [], + "Id": 562537974, + "Name": "from_address", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CData.from_address", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "obj", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:typing:Union[Unknown, ctypes(stub):_CArgObject]$typing(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 3, + "Classes": [], + "Functions": [], + "Id": -177816817, + "Name": "from_param", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CData.from_param", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "cls", + "Type": "t:typing:Type[Unknown]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "library", + "Type": "t:ctypes:CDLL$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": null, + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 3, + "Classes": [], + "Functions": [], + "Id": 2048326049, + "Name": "in_dll", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CData.in_dll", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1721516133, + "Name": "_CData", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CData", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_CData$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -638871764, + "Name": "_PointerLike", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_PointerLike", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_PointerLike$ctypes(3.7)", + "t:ctypes(stub):_CData$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "address", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "callable", + "Type": null, + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "func_spec", + "Type": "t:typing:Tuple[typing:Union[str, int], ctypes(stub):CDLL]$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "paramflags", + "Type": "t:typing:Tuple[Unknown]$typing(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + }, + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "vtlb_index", + "Type": "t:int", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "paramflags", + "Type": "t:typing:Tuple[Unknown]$typing(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + }, + { + "Name": "iid", + "Type": "t:ctypes(stub):pointer$ctypes(3.7)", + "DefaultValue": "i:...", + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_FuncPointer.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "args", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 1 + }, + { + "Name": "kwargs", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 2 + } + ], + "ReturnType": "i:typing:Any$typing(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 782173109, + "Name": "__call__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_FuncPointer.__call__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:typing:Union[typing:Type[ctypes(stub):_CData], Unknown, None]$typing(3.7)", + "Id": -1767505949, + "Name": "restype", + "DeclaringModuleId": null, + "QualifiedName": "restype", + "IndexSpan": null + }, + { + "Value": "i:typing:Sequence[typing:Type[ctypes(stub):_CData]]$typing(3.7)", + "Id": -212017926, + "Name": "argtypes", + "DeclaringModuleId": null, + "QualifiedName": "argtypes", + "IndexSpan": null + }, + { + "Value": null, + "Id": -1532007462, + "Name": "errcheck", + "DeclaringModuleId": null, + "QualifiedName": "errcheck", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -778081647, + "Name": "_FuncPointer", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_FuncPointer", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1648730790, + "Name": "_CArgObject", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CArgObject", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:object" + ], + "NamedTupleBases": [], + "Methods": [], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -2082031766, + "Name": "offset", + "DeclaringModuleId": null, + "QualifiedName": "offset", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": 24771736, + "Name": "size", + "DeclaringModuleId": null, + "QualifiedName": "size", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": 1829463967, + "Name": "_CField", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_CField", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": "type(object_or_name, bases, dict)\ntype(object) -> the object's type\ntype(name, bases, dict) -> a new type", + "Bases": [ + "t:ctypes(stub):_CDataMeta$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_StructUnionMeta$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:ctypes(stub):_CField$ctypes(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -521760752, + "Name": "__getattr__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_StructUnionMeta.__getattr__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:typing:Sequence[typing:Union[typing:Tuple[str, typing:Type[ctypes(stub):_CData]], typing:Tuple[str, typing:Type[ctypes(stub):_CData], int]]]$typing(3.7)", + "Id": -1419773794, + "Name": "_fields_", + "DeclaringModuleId": null, + "QualifiedName": "_fields_", + "IndexSpan": null + }, + { + "Value": "i:int", + "Id": 1763940158, + "Name": "_pack_", + "DeclaringModuleId": null, + "QualifiedName": "_pack_", + "IndexSpan": null + }, + { + "Value": "i:typing:Sequence[str]$typing(3.7)", + "Id": -783398214, + "Name": "_anonymous_", + "DeclaringModuleId": null, + "QualifiedName": "_anonymous_", + "IndexSpan": null + }, + { + "Value": null, + "Id": -1388753224, + "Name": "__basicsize__", + "DeclaringModuleId": null, + "QualifiedName": "__basicsize__", + "IndexSpan": null + }, + { + "Value": null, + "Id": -198224608, + "Name": "__dictoffset__", + "DeclaringModuleId": null, + "QualifiedName": "__dictoffset__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 1444705936, + "Name": "__flags__", + "DeclaringModuleId": null, + "QualifiedName": "__flags__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 919460331, + "Name": "__itemsize__", + "DeclaringModuleId": null, + "QualifiedName": "__itemsize__", + "IndexSpan": null + }, + { + "Value": "i:list$ctypes(3.7)", + "Id": -1627592461, + "Name": "__mro__", + "DeclaringModuleId": null, + "QualifiedName": "__mro__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 1097116834, + "Name": "__name__", + "DeclaringModuleId": null, + "QualifiedName": "__name__", + "IndexSpan": null + }, + { + "Value": null, + "Id": -1879833807, + "Name": "__qualname__", + "DeclaringModuleId": null, + "QualifiedName": "__qualname__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 174109373, + "Name": "__text_signature__", + "DeclaringModuleId": null, + "QualifiedName": "__text_signature__", + "IndexSpan": null + }, + { + "Value": null, + "Id": 1521523639, + "Name": "__weakrefoffset__", + "DeclaringModuleId": null, + "QualifiedName": "__weakrefoffset__", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1026689833, + "Name": "_StructUnionMeta", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_StructUnionMeta", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Documentation": null, + "Bases": [ + "t:ctypes(stub):_CData$ctypes(3.7)", + "t:object" + ], + "NamedTupleBases": [], + "Methods": [ + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "args", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 1 + }, + { + "Name": "kw", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 2 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": 965872103, + "Name": "__init__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_StructUnionBase.__init__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:typing:Any$typing(3.7)", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -521760752, + "Name": "__getattr__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_StructUnionBase.__getattr__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + }, + { + "Overloads": [ + { + "Parameters": [ + { + "Name": "self", + "Type": "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "name", + "Type": "t:str", + "DefaultValue": null, + "Kind": 0 + }, + { + "Name": "value", + "Type": "t:typing:Any$typing(3.7)", + "DefaultValue": null, + "Kind": 0 + } + ], + "ReturnType": "i:None", + "Documentation": null + } + ], + "Documentation": null, + "Attributes": 0, + "Classes": [], + "Functions": [], + "Id": -736377828, + "Name": "__setattr__", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_StructUnionBase.__setattr__", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "Properties": [], + "Fields": [ + { + "Value": "i:int", + "Id": -1555123786, + "Name": "_b_base", + "DeclaringModuleId": null, + "QualifiedName": "_b_base", + "IndexSpan": null + }, + { + "Value": "i:bool", + "Id": -422546917, + "Name": "_b_needsfree_", + "DeclaringModuleId": null, + "QualifiedName": "_b_needsfree_", + "IndexSpan": null + }, + { + "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", + "Id": 2076796140, + "Name": "_objects", + "DeclaringModuleId": null, + "QualifiedName": "_objects", + "IndexSpan": null + } + ], + "Classes": [], + "GenericBaseParameters": [], + "GenericParameterValues": [], + "Id": -1027021405, + "Name": "_StructUnionBase", + "DeclaringModuleId": "ctypes(3.7)", + "QualifiedName": "ctypes(stub):_StructUnionBase", + "IndexSpan": { + "Start": 0, + "Length": 0 + } + } + ], + "TypeVars": [ + { + "Constraints": [], + "Bound": null, + "Covariant": null, + "Contravariant": null, + "Id": 25132, + "Name": "_T", + "DeclaringModuleId": null, + "QualifiedName": "ctypes:_T", + "IndexSpan": null + }, + { + "Constraints": [], + "Bound": "ctypes:CDLL", + "Covariant": null, + "Contravariant": null, + "Id": 748306232, + "Name": "_DLLT", + "DeclaringModuleId": null, + "QualifiedName": "ctypes:_DLLT", + "IndexSpan": null + }, + { + "Constraints": [], + "Bound": "ctypes(stub):_CData", + "Covariant": null, + "Contravariant": null, + "Id": 778649, + "Name": "_CT", + "DeclaringModuleId": null, + "QualifiedName": "ctypes:_CT", + "IndexSpan": null + }, + { + "Constraints": [], + "Bound": "ctypes(stub):_PointerLike", + "Covariant": null, + "Contravariant": null, + "Id": 779052, + "Name": "_PT", + "DeclaringModuleId": null, + "QualifiedName": "ctypes:_PT", + "IndexSpan": null + } + ], + "NamedTuples": [], + "NewLines": [ + { + "EndIndex": 52, + "Kind": 3 + }, + { + "EndIndex": 54, + "Kind": 3 + }, + { + "EndIndex": 85, + "Kind": 3 + }, + { + "EndIndex": 87, + "Kind": 3 + }, + { + "EndIndex": 110, + "Kind": 3 + }, + { + "EndIndex": 112, + "Kind": 3 + }, + { + "EndIndex": 157, + "Kind": 3 + }, + { + "EndIndex": 187, + "Kind": 3 + }, + { + "EndIndex": 230, + "Kind": 3 + }, + { + "EndIndex": 282, + "Kind": 3 + }, + { + "EndIndex": 327, + "Kind": 3 + }, + { + "EndIndex": 362, + "Kind": 3 + }, + { + "EndIndex": 364, + "Kind": 3 + }, + { + "EndIndex": 406, + "Kind": 3 + }, + { + "EndIndex": 408, + "Kind": 3 + }, + { + "EndIndex": 444, + "Kind": 3 + }, + { + "EndIndex": 522, + "Kind": 3 + }, + { + "EndIndex": 524, + "Kind": 3 + }, + { + "EndIndex": 546, + "Kind": 3 + }, + { + "EndIndex": 583, + "Kind": 3 + }, + { + "EndIndex": 585, + "Kind": 3 + }, + { + "EndIndex": 612, + "Kind": 3 + }, + { + "EndIndex": 667, + "Kind": 3 + }, + { + "EndIndex": 723, + "Kind": 3 + }, + { + "EndIndex": 780, + "Kind": 3 + }, + { + "EndIndex": 838, + "Kind": 3 + }, + { + "EndIndex": 851, + "Kind": 3 + }, + { + "EndIndex": 853, + "Kind": 3 + }, + { + "EndIndex": 905, + "Kind": 3 + }, + { + "EndIndex": 941, + "Kind": 3 + }, + { + "EndIndex": 943, + "Kind": 3 + }, + { + "EndIndex": 1001, + "Kind": 3 + }, + { + "EndIndex": 1052, + "Kind": 3 + }, + { + "EndIndex": 1103, + "Kind": 3 + }, + { + "EndIndex": 1159, + "Kind": 3 + }, + { + "EndIndex": 1161, + "Kind": 3 + }, + { + "EndIndex": 1185, + "Kind": 3 + }, + { + "EndIndex": 1205, + "Kind": 3 + }, + { + "EndIndex": 1208, + "Kind": 3 + }, + { + "EndIndex": 1229, + "Kind": 3 + }, + { + "EndIndex": 1232, + "Kind": 3 + }, + { + "EndIndex": 1251, + "Kind": 3 + }, + { + "EndIndex": 1277, + "Kind": 3 + }, + { + "EndIndex": 1280, + "Kind": 3 + }, + { + "EndIndex": 1298, + "Kind": 3 + }, + { + "EndIndex": 1300, + "Kind": 3 + }, + { + "EndIndex": 1344, + "Kind": 3 + }, + { + "EndIndex": 1400, + "Kind": 3 + }, + { + "EndIndex": 1456, + "Kind": 3 + }, + { + "EndIndex": 1520, + "Kind": 3 + }, + { + "EndIndex": 1529, + "Kind": 3 + }, + { + "EndIndex": 1562, + "Kind": 3 + }, + { + "EndIndex": 1588, + "Kind": 3 + }, + { + "EndIndex": 1620, + "Kind": 3 + }, + { + "EndIndex": 1653, + "Kind": 3 + }, + { + "EndIndex": 1678, + "Kind": 3 + }, + { + "EndIndex": 1704, + "Kind": 3 + }, + { + "EndIndex": 1724, + "Kind": 3 + }, + { + "EndIndex": 1757, + "Kind": 3 + }, + { + "EndIndex": 1790, + "Kind": 3 + }, + { + "EndIndex": 1815, + "Kind": 3 + }, + { + "EndIndex": 1835, + "Kind": 3 + }, + { + "EndIndex": 1862, + "Kind": 3 + }, + { + "EndIndex": 1864, + "Kind": 3 + }, + { + "EndIndex": 1896, + "Kind": 3 + }, + { + "EndIndex": 1950, + "Kind": 3 + }, + { + "EndIndex": 1973, + "Kind": 3 + }, + { + "EndIndex": 2054, + "Kind": 3 + }, + { + "EndIndex": 2109, + "Kind": 3 + }, + { + "EndIndex": 2154, + "Kind": 3 + }, + { + "EndIndex": 2156, + "Kind": 3 + }, + { + "EndIndex": 2180, + "Kind": 3 + }, + { + "EndIndex": 2222, + "Kind": 3 + }, + { + "EndIndex": 2260, + "Kind": 3 + }, + { + "EndIndex": 2340, + "Kind": 3 + }, + { + "EndIndex": 2342, + "Kind": 3 + }, + { + "EndIndex": 2372, + "Kind": 3 + }, + { + "EndIndex": 2428, + "Kind": 3 + }, + { + "EndIndex": 2430, + "Kind": 3 + }, + { + "EndIndex": 2502, + "Kind": 3 + }, + { + "EndIndex": 2524, + "Kind": 3 + }, + { + "EndIndex": 2526, + "Kind": 3 + }, + { + "EndIndex": 2578, + "Kind": 3 + }, + { + "EndIndex": 2660, + "Kind": 3 + }, + { + "EndIndex": 2758, + "Kind": 3 + }, + { + "EndIndex": 2857, + "Kind": 3 + }, + { + "EndIndex": 2952, + "Kind": 3 + }, + { + "EndIndex": 2961, + "Kind": 3 + }, + { + "EndIndex": 2990, + "Kind": 3 + }, + { + "EndIndex": 3026, + "Kind": 3 + }, + { + "EndIndex": 3064, + "Kind": 3 + }, + { + "EndIndex": 3105, + "Kind": 3 + }, + { + "EndIndex": 3147, + "Kind": 3 + }, + { + "EndIndex": 3159, + "Kind": 3 + }, + { + "EndIndex": 3234, + "Kind": 3 + }, + { + "EndIndex": 3244, + "Kind": 3 + }, + { + "EndIndex": 3306, + "Kind": 3 + }, + { + "EndIndex": 3328, + "Kind": 3 + }, + { + "EndIndex": 3369, + "Kind": 3 + }, + { + "EndIndex": 3404, + "Kind": 3 + }, + { + "EndIndex": 3437, + "Kind": 3 + }, + { + "EndIndex": 3466, + "Kind": 3 + }, + { + "EndIndex": 3537, + "Kind": 3 + }, + { + "EndIndex": 3567, + "Kind": 3 + }, + { + "EndIndex": 3569, + "Kind": 3 + }, + { + "EndIndex": 3591, + "Kind": 3 + }, + { + "EndIndex": 3639, + "Kind": 3 + }, + { + "EndIndex": 3702, + "Kind": 3 + }, + { + "EndIndex": 3704, + "Kind": 3 + }, + { + "EndIndex": 3734, + "Kind": 3 + }, + { + "EndIndex": 3782, + "Kind": 3 + }, + { + "EndIndex": 3849, + "Kind": 3 + }, + { + "EndIndex": 3884, + "Kind": 3 + }, + { + "EndIndex": 3924, + "Kind": 3 + }, + { + "EndIndex": 3966, + "Kind": 3 + }, + { + "EndIndex": 4011, + "Kind": 3 + }, + { + "EndIndex": 4057, + "Kind": 3 + }, + { + "EndIndex": 4073, + "Kind": 3 + }, + { + "EndIndex": 4152, + "Kind": 3 + }, + { + "EndIndex": 4166, + "Kind": 3 + }, + { + "EndIndex": 4234, + "Kind": 3 + }, + { + "EndIndex": 4260, + "Kind": 3 + }, + { + "EndIndex": 4307, + "Kind": 3 + }, + { + "EndIndex": 4346, + "Kind": 3 + }, + { + "EndIndex": 4383, + "Kind": 3 + }, + { + "EndIndex": 4416, + "Kind": 3 + }, + { + "EndIndex": 4495, + "Kind": 3 + }, + { + "EndIndex": 4531, + "Kind": 3 + }, + { + "EndIndex": 4560, + "Kind": 3 + }, + { + "EndIndex": 4645, + "Kind": 3 + }, + { + "EndIndex": 4647, + "Kind": 3 + }, + { + "EndIndex": 4674, + "Kind": 3 + }, + { + "EndIndex": 4717, + "Kind": 3 + }, + { + "EndIndex": 4719, + "Kind": 3 + }, + { + "EndIndex": 4784, + "Kind": 3 + }, + { + "EndIndex": 4826, + "Kind": 3 + }, + { + "EndIndex": 4860, + "Kind": 3 + }, + { + "EndIndex": 4862, + "Kind": 3 + }, + { + "EndIndex": 4900, + "Kind": 3 + }, + { + "EndIndex": 4967, + "Kind": 3 + }, + { + "EndIndex": 5030, + "Kind": 3 + }, + { + "EndIndex": 5063, + "Kind": 3 + }, + { + "EndIndex": 5089, + "Kind": 3 + }, + { + "EndIndex": 5149, + "Kind": 3 + }, + { + "EndIndex": 5180, + "Kind": 3 + }, + { + "EndIndex": 5236, + "Kind": 3 + }, + { + "EndIndex": 5264, + "Kind": 3 + }, + { + "EndIndex": 5332, + "Kind": 3 + }, + { + "EndIndex": 5384, + "Kind": 3 + }, + { + "EndIndex": 5386, + "Kind": 3 + }, + { + "EndIndex": 5418, + "Kind": 3 + }, + { + "EndIndex": 5436, + "Kind": 3 + }, + { + "EndIndex": 5461, + "Kind": 3 + }, + { + "EndIndex": 5475, + "Kind": 3 + }, + { + "EndIndex": 5514, + "Kind": 3 + }, + { + "EndIndex": 5542, + "Kind": 3 + }, + { + "EndIndex": 5597, + "Kind": 3 + }, + { + "EndIndex": 5626, + "Kind": 3 + }, + { + "EndIndex": 5628, + "Kind": 3 + }, + { + "EndIndex": 5658, + "Kind": 3 + }, + { + "EndIndex": 5676, + "Kind": 3 + }, + { + "EndIndex": 5698, + "Kind": 3 + }, + { + "EndIndex": 5700, + "Kind": 3 + }, + { + "EndIndex": 5731, + "Kind": 3 + }, + { + "EndIndex": 5749, + "Kind": 3 + }, + { + "EndIndex": 5772, + "Kind": 3 + }, + { + "EndIndex": 5774, + "Kind": 3 + }, + { + "EndIndex": 5803, + "Kind": 3 + }, + { + "EndIndex": 5821, + "Kind": 3 + }, + { + "EndIndex": 5842, + "Kind": 3 + }, + { + "EndIndex": 5844, + "Kind": 3 + }, + { + "EndIndex": 5874, + "Kind": 3 + }, + { + "EndIndex": 5892, + "Kind": 3 + }, + { + "EndIndex": 5914, + "Kind": 3 + }, + { + "EndIndex": 5916, + "Kind": 3 + }, + { + "EndIndex": 5954, + "Kind": 3 + }, + { + "EndIndex": 6028, + "Kind": 3 + }, + { + "EndIndex": 6048, + "Kind": 3 + }, + { + "EndIndex": 6070, + "Kind": 3 + }, + { + "EndIndex": 6077, + "Kind": 3 + }, + { + "EndIndex": 6109, + "Kind": 3 + }, + { + "EndIndex": 6131, + "Kind": 3 + }, + { + "EndIndex": 6155, + "Kind": 3 + }, + { + "EndIndex": 6157, + "Kind": 3 + }, + { + "EndIndex": 6190, + "Kind": 3 + }, + { + "EndIndex": 6212, + "Kind": 3 + }, + { + "EndIndex": 6237, + "Kind": 3 + }, + { + "EndIndex": 6239, + "Kind": 3 + }, + { + "EndIndex": 6269, + "Kind": 3 + }, + { + "EndIndex": 6287, + "Kind": 3 + }, + { + "EndIndex": 6309, + "Kind": 3 + }, + { + "EndIndex": 6311, + "Kind": 3 + }, + { + "EndIndex": 6342, + "Kind": 3 + }, + { + "EndIndex": 6360, + "Kind": 3 + }, + { + "EndIndex": 6383, + "Kind": 3 + }, + { + "EndIndex": 6385, + "Kind": 3 + }, + { + "EndIndex": 6420, + "Kind": 3 + }, + { + "EndIndex": 6438, + "Kind": 3 + }, + { + "EndIndex": 6484, + "Kind": 3 + }, + { + "EndIndex": 6513, + "Kind": 3 + }, + { + "EndIndex": 6515, + "Kind": 3 + }, + { + "EndIndex": 6553, + "Kind": 3 + }, + { + "EndIndex": 6638, + "Kind": 3 + }, + { + "EndIndex": 6663, + "Kind": 3 + }, + { + "EndIndex": 6690, + "Kind": 3 + }, + { + "EndIndex": 6697, + "Kind": 3 + }, + { + "EndIndex": 6734, + "Kind": 3 + }, + { + "EndIndex": 6756, + "Kind": 3 + }, + { + "EndIndex": 6785, + "Kind": 3 + }, + { + "EndIndex": 6787, + "Kind": 3 + }, + { + "EndIndex": 6825, + "Kind": 3 + }, + { + "EndIndex": 6847, + "Kind": 3 + }, + { + "EndIndex": 6884, + "Kind": 3 + }, + { + "EndIndex": 6929, + "Kind": 3 + }, + { + "EndIndex": 6977, + "Kind": 3 + }, + { + "EndIndex": 7007, + "Kind": 3 + }, + { + "EndIndex": 7009, + "Kind": 3 + }, + { + "EndIndex": 7039, + "Kind": 3 + }, + { + "EndIndex": 7057, + "Kind": 3 + }, + { + "EndIndex": 7112, + "Kind": 3 + }, + { + "EndIndex": 7139, + "Kind": 3 + }, + { + "EndIndex": 7160, + "Kind": 3 + }, + { + "EndIndex": 7182, + "Kind": 3 + }, + { + "EndIndex": 7184, + "Kind": 3 + }, + { + "EndIndex": 7213, + "Kind": 3 + }, + { + "EndIndex": 7231, + "Kind": 3 + }, + { + "EndIndex": 7283, + "Kind": 3 + }, + { + "EndIndex": 7304, + "Kind": 3 + }, + { + "EndIndex": 7306, + "Kind": 3 + }, + { + "EndIndex": 7335, + "Kind": 3 + }, + { + "EndIndex": 7353, + "Kind": 3 + }, + { + "EndIndex": 7405, + "Kind": 3 + }, + { + "EndIndex": 7426, + "Kind": 3 + }, + { + "EndIndex": 7428, + "Kind": 3 + }, + { + "EndIndex": 7459, + "Kind": 3 + }, + { + "EndIndex": 7477, + "Kind": 3 + }, + { + "EndIndex": 7502, + "Kind": 3 + }, + { + "EndIndex": 7589, + "Kind": 3 + }, + { + "EndIndex": 7617, + "Kind": 3 + }, + { + "EndIndex": 7619, + "Kind": 3 + }, + { + "EndIndex": 7650, + "Kind": 3 + }, + { + "EndIndex": 7668, + "Kind": 3 + }, + { + "EndIndex": 7725, + "Kind": 3 + }, + { + "EndIndex": 7748, + "Kind": 3 + }, + { + "EndIndex": 7750, + "Kind": 3 + }, + { + "EndIndex": 7779, + "Kind": 3 + }, + { + "EndIndex": 7797, + "Kind": 3 + }, + { + "EndIndex": 7799, + "Kind": 3 + }, + { + "EndIndex": 7858, + "Kind": 3 + }, + { + "EndIndex": 7860, + "Kind": 3 + }, + { + "EndIndex": 7892, + "Kind": 3 + }, + { + "EndIndex": 7910, + "Kind": 3 + }, + { + "EndIndex": 7935, + "Kind": 3 + }, + { + "EndIndex": 8022, + "Kind": 3 + }, + { + "EndIndex": 8024, + "Kind": 3 + }, + { + "EndIndex": 8054, + "Kind": 3 + }, + { + "EndIndex": 8072, + "Kind": 3 + }, + { + "EndIndex": 8074, + "Kind": 3 + }, + { + "EndIndex": 8095, + "Kind": 3 + }, + { + "EndIndex": 8128, + "Kind": 3 + }, + { + "EndIndex": 8159, + "Kind": 3 + }, + { + "EndIndex": 8185, + "Kind": 3 + }, + { + "EndIndex": 8222, + "Kind": 3 + }, + { + "EndIndex": 8263, + "Kind": 3 + }, + { + "EndIndex": 8319, + "Kind": 3 + }, + { + "EndIndex": 8359, + "Kind": 3 + }, + { + "EndIndex": 8413, + "Kind": 3 + }, + { + "EndIndex": 8455, + "Kind": 3 + }, + { + "EndIndex": 8530, + "Kind": 3 + }, + { + "EndIndex": 8605, + "Kind": 3 + }, + { + "EndIndex": 8677, + "Kind": 3 + }, + { + "EndIndex": 8750, + "Kind": 3 + }, + { + "EndIndex": 8786, + "Kind": 3 + }, + { + "EndIndex": 8788, + "Kind": 3 + }, + { + "EndIndex": 8833, + "Kind": 3 + }, + { + "EndIndex": 8891, + "Kind": 3 + }, + { + "EndIndex": 8948, + "Kind": 3 + }, + { + "EndIndex": 9014, + "Kind": 3 + }, + { + "EndIndex": 9023, + "Kind": 3 + }, + { + "EndIndex": 9054, + "Kind": 3 + }, + { + "EndIndex": 9080, + "Kind": 3 + }, + { + "EndIndex": 9112, + "Kind": 3 + }, + { + "EndIndex": 9146, + "Kind": 3 + }, + { + "EndIndex": 9171, + "Kind": 3 + }, + { + "EndIndex": 9197, + "Kind": 3 + }, + { + "EndIndex": 9217, + "Kind": 3 + }, + { + "EndIndex": 9250, + "Kind": 3 + }, + { + "EndIndex": 9284, + "Kind": 3 + }, + { + "EndIndex": 9309, + "Kind": 3 + }, + { + "EndIndex": 9329, + "Kind": 3 + }, + { + "EndIndex": 9356, + "Kind": 3 + }, + { + "EndIndex": 9358, + "Kind": 3 + }, + { + "EndIndex": 9360, + "Kind": 3 + }, + { + "EndIndex": 9378, + "Kind": 3 + }, + { + "EndIndex": 9413, + "Kind": 3 + }, + { + "EndIndex": 9469, + "Kind": 3 + }, + { + "EndIndex": 9538, + "Kind": 3 + }, + { + "EndIndex": 9586, + "Kind": 3 + }, + { + "EndIndex": 9632, + "Kind": 3 + }, + { + "EndIndex": 9659, + "Kind": 3 + }, + { + "EndIndex": 9699, + "Kind": 3 + }, + { + "EndIndex": 9741, + "Kind": 3 + }, + { + "EndIndex": 9743, + "Kind": 3 + }, + { + "EndIndex": 9761, + "Kind": 3 + }, + { + "EndIndex": 9783, + "Kind": 3 + }, + { + "EndIndex": 9805, + "Kind": 3 + }, + { + "EndIndex": 9807, + "Kind": 3 + }, + { + "EndIndex": 9873, + "Kind": 3 + }, + { + "EndIndex": 9875, + "Kind": 3 + }, + { + "EndIndex": 9877, + "Kind": 3 + }, + { + "EndIndex": 9898, + "Kind": 3 + }, + { + "EndIndex": 9963, + "Kind": 3 + }, + { + "EndIndex": 10026, + "Kind": 3 + }, + { + "EndIndex": 10070, + "Kind": 3 + }, + { + "EndIndex": 10072, + "Kind": 3 + }, + { + "EndIndex": 10137, + "Kind": 3 + }, + { + "EndIndex": 10186, + "Kind": 3 + }, + { + "EndIndex": 10188, + "Kind": 3 + }, + { + "EndIndex": 10224, + "Kind": 3 + }, + { + "EndIndex": 10263, + "Kind": 3 + }, + { + "EndIndex": 10265, + "Kind": 3 + }, + { + "EndIndex": 10336, + "Kind": 3 + }, + { + "EndIndex": 10367, + "Kind": 3 + }, + { + "EndIndex": 10376, + "Kind": 3 + }, + { + "EndIndex": 10412, + "Kind": 3 + }, + { + "EndIndex": 10440, + "Kind": 3 + }, + { + "EndIndex": 10471, + "Kind": 3 + }, + { + "EndIndex": 10502, + "Kind": 3 + }, + { + "EndIndex": 10519, + "Kind": 3 + }, + { + "EndIndex": 10540, + "Kind": 3 + }, + { + "EndIndex": 10542, + "Kind": 3 + }, + { + "EndIndex": 10604, + "Kind": 3 + }, + { + "EndIndex": 10639, + "Kind": 3 + }, + { + "EndIndex": 10680, + "Kind": 3 + }, + { + "EndIndex": 10707, + "Kind": 3 + }, + { + "EndIndex": 10742, + "Kind": 3 + }, + { + "EndIndex": 10765, + "Kind": 3 + }, + { + "EndIndex": 10807, + "Kind": 3 + }, + { + "EndIndex": 10835, + "Kind": 3 + }, + { + "EndIndex": 10881, + "Kind": 3 + }, + { + "EndIndex": 10926, + "Kind": 3 + }, + { + "EndIndex": 10990, + "Kind": 3 + }, + { + "EndIndex": 11059, + "Kind": 3 + }, + { + "EndIndex": 11138, + "Kind": 3 + }, + { + "EndIndex": 11215, + "Kind": 3 + }, + { + "EndIndex": 11232, + "Kind": 3 + }, + { + "EndIndex": 11295, + "Kind": 3 + }, + { + "EndIndex": 11355, + "Kind": 3 + }, + { + "EndIndex": 11357, + "Kind": 3 + }, + { + "EndIndex": 11393, + "Kind": 3 + }, + { + "EndIndex": 11422, + "Kind": 3 + }, + { + "EndIndex": 11467, + "Kind": 3 + }, + { + "EndIndex": 11501, + "Kind": 3 + }, + { + "EndIndex": 11503, + "Kind": 3 + }, + { + "EndIndex": 11531, + "Kind": 3 + }, + { + "EndIndex": 11585, + "Kind": 3 + }, + { + "EndIndex": 11600, + "Kind": 3 + }, + { + "EndIndex": 11635, + "Kind": 3 + }, + { + "EndIndex": 11637, + "Kind": 3 + }, + { + "EndIndex": 11662, + "Kind": 3 + }, + { + "EndIndex": 11712, + "Kind": 3 + }, + { + "EndIndex": 11766, + "Kind": 3 + }, + { + "EndIndex": 11822, + "Kind": 3 + }, + { + "EndIndex": 11872, + "Kind": 3 + }, + { + "EndIndex": 11874, + "Kind": 3 + }, + { + "EndIndex": 11908, + "Kind": 3 + }, + { + "EndIndex": 11967, + "Kind": 3 + }, + { + "EndIndex": 12007, + "Kind": 3 + }, + { + "EndIndex": 12046, + "Kind": 3 + }, + { + "EndIndex": 12081, + "Kind": 3 + }, + { + "EndIndex": 12102, + "Kind": 3 + }, + { + "EndIndex": 12104, + "Kind": 3 + }, + { + "EndIndex": 12149, + "Kind": 3 + }, + { + "EndIndex": 12204, + "Kind": 3 + }, + { + "EndIndex": 12254, + "Kind": 3 + }, + { + "EndIndex": 12299, + "Kind": 3 + }, + { + "EndIndex": 12320, + "Kind": 3 + }, + { + "EndIndex": 12322, + "Kind": 3 + }, + { + "EndIndex": 12342, + "Kind": 3 + }, + { + "EndIndex": 12410, + "Kind": 3 + }, + { + "EndIndex": 12477, + "Kind": 3 + }, + { + "EndIndex": 12523, + "Kind": 3 + }, + { + "EndIndex": 12532, + "Kind": 3 + }, + { + "EndIndex": 12590, + "Kind": 3 + }, + { + "EndIndex": 12592, + "Kind": 3 + }, + { + "EndIndex": 12614, + "Kind": 3 + }, + { + "EndIndex": 12616, + "Kind": 3 + }, + { + "EndIndex": 12641, + "Kind": 3 + }, + { + "EndIndex": 12711, + "Kind": 3 + }, + { + "EndIndex": 12756, + "Kind": 3 + }, + { + "EndIndex": 12769, + "Kind": 3 + }, + { + "EndIndex": 12811, + "Kind": 3 + }, + { + "EndIndex": 12813, + "Kind": 3 + }, + { + "EndIndex": 12868, + "Kind": 3 + }, + { + "EndIndex": 12911, + "Kind": 3 + }, + { + "EndIndex": 12965, + "Kind": 3 + }, + { + "EndIndex": 12999, + "Kind": 3 + }, + { + "EndIndex": 13021, + "Kind": 3 + }, + { + "EndIndex": 13092, + "Kind": 3 + }, + { + "EndIndex": 13158, + "Kind": 3 + }, + { + "EndIndex": 13201, + "Kind": 3 + }, + { + "EndIndex": 13212, + "Kind": 3 + }, + { + "EndIndex": 13282, + "Kind": 3 + }, + { + "EndIndex": 13351, + "Kind": 3 + }, + { + "EndIndex": 13421, + "Kind": 3 + }, + { + "EndIndex": 13489, + "Kind": 3 + }, + { + "EndIndex": 13508, + "Kind": 3 + }, + { + "EndIndex": 13549, + "Kind": 3 + }, + { + "EndIndex": 13551, + "Kind": 3 + }, + { + "EndIndex": 13576, + "Kind": 3 + }, + { + "EndIndex": 13646, + "Kind": 3 + }, + { + "EndIndex": 13714, + "Kind": 3 + }, + { + "EndIndex": 13780, + "Kind": 3 + }, + { + "EndIndex": 13801, + "Kind": 3 + }, + { + "EndIndex": 13814, + "Kind": 3 + }, + { + "EndIndex": 13856, + "Kind": 3 + }, + { + "EndIndex": 13890, + "Kind": 3 + }, + { + "EndIndex": 13892, + "Kind": 3 + }, + { + "EndIndex": 13922, + "Kind": 3 + }, + { + "EndIndex": 13956, + "Kind": 3 + }, + { + "EndIndex": 13989, + "Kind": 3 + }, + { + "EndIndex": 13991, + "Kind": 3 + }, + { + "EndIndex": 14025, + "Kind": 3 + }, + { + "EndIndex": 14053, + "Kind": 3 + }, + { + "EndIndex": 14093, + "Kind": 3 + }, + { + "EndIndex": 14128, + "Kind": 3 + }, + { + "EndIndex": 14162, + "Kind": 3 + }, + { + "EndIndex": 14182, + "Kind": 3 + }, + { + "EndIndex": 14184, + "Kind": 3 + }, + { + "EndIndex": 14218, + "Kind": 3 + }, + { + "EndIndex": 14254, + "Kind": 3 + }, + { + "EndIndex": 14256, + "Kind": 3 + }, + { + "EndIndex": 14290, + "Kind": 3 + }, + { + "EndIndex": 14326, + "Kind": 3 + }, + { + "EndIndex": 14328, + "Kind": 3 + }, + { + "EndIndex": 14356, + "Kind": 3 + }, + { + "EndIndex": 14386, + "Kind": 3 + }, + { + "EndIndex": 14388, + "Kind": 3 + }, + { + "EndIndex": 14410, + "Kind": 3 + }, + { + "EndIndex": 14469, + "Kind": 3 + }, + { + "EndIndex": 14502, + "Kind": 3 + }, + { + "EndIndex": 14571, + "Kind": 3 + }, + { + "EndIndex": 14578, + "Kind": 3 + }, + { + "EndIndex": 14607, + "Kind": 3 + }, + { + "EndIndex": 14609, + "Kind": 3 + }, + { + "EndIndex": 14611, + "Kind": 3 + }, + { + "EndIndex": 14633, + "Kind": 3 + }, + { + "EndIndex": 14669, + "Kind": 3 + }, + { + "EndIndex": 14705, + "Kind": 3 + }, + { + "EndIndex": 14707, + "Kind": 3 + }, + { + "EndIndex": 14756, + "Kind": 3 + }, + { + "EndIndex": 14812, + "Kind": 3 + }, + { + "EndIndex": 14814, + "Kind": 3 + }, + { + "EndIndex": 14856, + "Kind": 3 + }, + { + "EndIndex": 14882, + "Kind": 3 + }, + { + "EndIndex": 14917, + "Kind": 3 + }, + { + "EndIndex": 14944, + "Kind": 3 + }, + { + "EndIndex": 14991, + "Kind": 3 + }, + { + "EndIndex": 15040, + "Kind": 3 + }, + { + "EndIndex": 15042, + "Kind": 3 + }, + { + "EndIndex": 15082, + "Kind": 3 + }, + { + "EndIndex": 15105, + "Kind": 3 + }, + { + "EndIndex": 15128, + "Kind": 3 + }, + { + "EndIndex": 15171, + "Kind": 3 + }, + { + "EndIndex": 15195, + "Kind": 3 + }, + { + "EndIndex": 15219, + "Kind": 3 + }, + { + "EndIndex": 15266, + "Kind": 3 + }, + { + "EndIndex": 15294, + "Kind": 3 + }, + { + "EndIndex": 15322, + "Kind": 3 + }, + { + "EndIndex": 15324, + "Kind": 3 + }, + { + "EndIndex": 15337, + "Kind": 3 + }, + { + "EndIndex": 15339, + "Kind": 3 + }, + { + "EndIndex": 15417, + "Kind": 3 + }, + { + "EndIndex": 15419, + "Kind": 3 + }, + { + "EndIndex": 15468, + "Kind": 3 + }, + { + "EndIndex": 15544, + "Kind": 3 + }, + { + "EndIndex": 15546, + "Kind": 3 + }, + { + "EndIndex": 15584, + "Kind": 3 + }, + { + "EndIndex": 15655, + "Kind": 3 + }, + { + "EndIndex": 15657, + "Kind": 3 + }, + { + "EndIndex": 15694, + "Kind": 3 + }, + { + "EndIndex": 15731, + "Kind": 3 + }, + { + "EndIndex": 15762, + "Kind": 3 + }, + { + "EndIndex": 15791, + "Kind": 3 + }, + { + "EndIndex": 15848, + "Kind": 3 + }, + { + "EndIndex": 15874, + "Kind": 3 + }, + { + "EndIndex": 15876, + "Kind": 3 + }, + { + "EndIndex": 15951, + "Kind": 3 + }, + { + "EndIndex": 15972, + "Kind": 3 + }, + { + "EndIndex": 16005, + "Kind": 3 + }, + { + "EndIndex": 16007, + "Kind": 3 + }, + { + "EndIndex": 16077, + "Kind": 3 + }, + { + "EndIndex": 16107, + "Kind": 3 + }, + { + "EndIndex": 16149, + "Kind": 3 + }, + { + "EndIndex": 16151, + "Kind": 3 + }, + { + "EndIndex": 16186, + "Kind": 3 + }, + { + "EndIndex": 16220, + "Kind": 3 + }, + { + "EndIndex": 16222, + "Kind": 3 + }, + { + "EndIndex": 16228, + "Kind": 3 + }, + { + "EndIndex": 16270, + "Kind": 3 + }, + { + "EndIndex": 16291, + "Kind": 3 + }, + { + "EndIndex": 16301, + "Kind": 3 + }, + { + "EndIndex": 16308, + "Kind": 3 + }, + { + "EndIndex": 16384, + "Kind": 3 + }, + { + "EndIndex": 16419, + "Kind": 3 + }, + { + "EndIndex": 16466, + "Kind": 3 + }, + { + "EndIndex": 16468, + "Kind": 3 + }, + { + "EndIndex": 16507, + "Kind": 3 + }, + { + "EndIndex": 16546, + "Kind": 3 + }, + { + "EndIndex": 16548, + "Kind": 3 + }, + { + "EndIndex": 16550, + "Kind": 3 + }, + { + "EndIndex": 16584, + "Kind": 3 + }, + { + "EndIndex": 16631, + "Kind": 3 + }, + { + "EndIndex": 16645, + "Kind": 3 + }, + { + "EndIndex": 16736, + "Kind": 3 + }, + { + "EndIndex": 16765, + "Kind": 3 + }, + { + "EndIndex": 16825, + "Kind": 3 + }, + { + "EndIndex": 16840, + "Kind": 3 + }, + { + "EndIndex": 16902, + "Kind": 3 + }, + { + "EndIndex": 16904, + "Kind": 3 + }, + { + "EndIndex": 16932, + "Kind": 3 + }, + { + "EndIndex": 16946, + "Kind": 3 + }, + { + "EndIndex": 17037, + "Kind": 3 + }, + { + "EndIndex": 17066, + "Kind": 3 + }, + { + "EndIndex": 17095, + "Kind": 3 + }, + { + "EndIndex": 17134, + "Kind": 3 + }, + { + "EndIndex": 17136, + "Kind": 3 + }, + { + "EndIndex": 17206, + "Kind": 3 + }, + { + "EndIndex": 17208, + "Kind": 3 + }, + { + "EndIndex": 17244, + "Kind": 3 + }, + { + "EndIndex": 17261, + "Kind": 3 + }, + { + "EndIndex": 17280, + "Kind": 3 + }, + { + "EndIndex": 17331, + "Kind": 3 + }, + { + "EndIndex": 17373, + "Kind": 3 + }, + { + "EndIndex": 17417, + "Kind": 3 + }, + { + "EndIndex": 17461, + "Kind": 3 + }, + { + "EndIndex": 17516, + "Kind": 3 + }, + { + "EndIndex": 17559, + "Kind": 3 + }, + { + "EndIndex": 17604, + "Kind": 3 + }, + { + "EndIndex": 17649, + "Kind": 3 + }, + { + "EndIndex": 17660, + "Kind": 3 + }, + { + "EndIndex": 17662, + "Kind": 3 + }, + { + "EndIndex": 17678, + "Kind": 3 + } + ], + "FileSize": 17678, + "Id": 585668072, + "Name": "ctypes", + "DeclaringModuleId": null, + "QualifiedName": "ctypes", + "IndexSpan": null +} \ No newline at end of file From 7a488b3206cbbf901eb280d8816ee0c0f92607fa Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sun, 10 Nov 2019 14:12:52 -0800 Subject: [PATCH 036/102] Don't restore all members on GetMember. Improves tensorflow by 30% --- src/Caching/Impl/Lazy/PythonLazyClassType.cs | 11 ++++------- src/Caching/Impl/Lazy/PythonLazyFunctionType.cs | 6 ++++-- src/Caching/Impl/Lazy/PythonLazyPropertyType.cs | 4 ++++ src/Caching/Impl/Lazy/PythonLazyType.cs | 16 +++++++++++----- src/Caching/Impl/ModuleFactory.cs | 4 ++-- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/Caching/Impl/Lazy/PythonLazyClassType.cs b/src/Caching/Impl/Lazy/PythonLazyClassType.cs index 466dc6fb0..27f9a612b 100644 --- a/src/Caching/Impl/Lazy/PythonLazyClassType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyClassType.cs @@ -95,13 +95,7 @@ protected override void EnsureContent(ClassModel cm) { ); } - var allMemberModels = cm.Classes - .Concat(cm.Properties) - .Concat(cm.Methods) - .Concat(cm.Fields) - .ToArray(); - - foreach (var model in allMemberModels) { + foreach (var model in GetMemberModels(cm)) { _cls.AddMember(model.Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _cls), false); } _cls.AddMember("__class__", _cls, true); @@ -146,5 +140,8 @@ private IEnumerable CreateBases(ClassModel cm, ModuleFactory mf, IG _cls.AddMember("__bases__", PythonCollectionType.CreateList(DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule, bases), true); return bases; } + + protected override IEnumerable GetMemberModels(ClassModel cm) + => cm.Classes.Concat(cm.Properties).Concat(cm.Methods).Concat(cm.Fields); } } diff --git a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs index 352376bdc..527c45b03 100644 --- a/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyFunctionType.cs @@ -49,10 +49,12 @@ public PythonLazyFunctionType(FunctionModel model, ModuleFactory mf, IGlobalScop #endregion protected override void EnsureContent(FunctionModel fm) { - var innerTypes = fm.Classes.Concat(fm.Functions).ToArray(); - foreach (var model in innerTypes) { + foreach (var model in GetMemberModels(fm)) { _function.AddMember(model.Name, MemberFactory.CreateMember(model, ModuleFactory, GlobalScope, _function), overwrite: true); } } + + protected override IEnumerable GetMemberModels(FunctionModel fm) + => fm.Classes.Concat(fm.Functions); } } diff --git a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs index def0791b7..47d605ab8 100644 --- a/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyPropertyType.cs @@ -13,6 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; +using System.Linq; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -51,5 +53,7 @@ public IMember ReturnType { protected override void EnsureContent(PropertyModel pm) { _property.Getter.SetReturnValue(ModuleFactory.ConstructMember(pm.ReturnType), true); } + protected override IEnumerable GetMemberModels(PropertyModel pm) + => pm.Classes.Concat(pm.Functions); } } diff --git a/src/Caching/Impl/Lazy/PythonLazyType.cs b/src/Caching/Impl/Lazy/PythonLazyType.cs index 2576d19ce..421fbf9fc 100644 --- a/src/Caching/Impl/Lazy/PythonLazyType.cs +++ b/src/Caching/Impl/Lazy/PythonLazyType.cs @@ -15,8 +15,11 @@ using System; using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching.Lazy { /// @@ -41,15 +44,17 @@ protected PythonLazyType(TModel model, ModuleFactory mf, IGlobalScope gs, IPytho public IPythonType DeclaringType { get; } #region IPythonType + public override IMember GetMember(string name) { - EnsureContent(); + if (_model != null) { + var memberModel = GetMemberModels(_model).FirstOrDefault(m => m.Name == name); + return memberModel != null ? MemberFactory.CreateMember(memberModel, ModuleFactory, GlobalScope, this) : null; + } return base.GetMember(name); } - public override IEnumerable GetMemberNames() { - EnsureContent(); - return base.GetMemberNames(); - } + public override IEnumerable GetMemberNames() + => _model != null ? GetMemberModels(_model).Select(m => m.Name) : base.GetMemberNames(); #endregion internal void EnsureContent() { @@ -65,5 +70,6 @@ internal void EnsureContent() { } protected abstract void EnsureContent(TModel model); + protected abstract IEnumerable GetMemberModels(TModel m); } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index a4e138fe6..ac53df9a0 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -202,12 +202,12 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { } if (member == null) { - var containerName = mc is IPythonType t ? t.Name : ""; + //var containerName = mc is IPythonType t ? t.Name : ""; //Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); break; } - member = typeArgs.Count > 0 && member is IGenericType gt + member = typeArgs.Count > 0 && member is IGenericType gt && typeArgs.Any(a => !(a is IGenericTypeParameter)) ? gt.CreateSpecificType(new ArgumentSet(typeArgs, null, null)) : member; } From c5f2cb8fcbdc56b1b588085682e59f2e7408e4e7 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sun, 10 Nov 2019 16:16:34 -0800 Subject: [PATCH 037/102] Turn off indexing of libraries, 25% --- src/LanguageServer/Impl/Indexing/IndexManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index 8d3595d5b..68d679fb4 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -78,7 +78,7 @@ public Task IndexWorkspace(PathResolverSnapshot snapshot = null, CancellationTok CreateIndices(userFiles, _userCodeSymbolIndex, linkedCt); // index library files if asked - CreateIndices(LibraryFiles(snapshot).Except(userFiles, FileSystemInfoComparer.Instance), _libraryCodeSymbolIndex, linkedCt); + // CreateIndices(LibraryFiles(snapshot).Except(userFiles, FileSystemInfoComparer.Instance), _libraryCodeSymbolIndex, linkedCt); }, linkedCt).ContinueWith(_ => linkedCts.Dispose()); } From dee842076ce78316277e380296e6204b66f86c3e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sun, 10 Nov 2019 16:50:12 -0800 Subject: [PATCH 038/102] optimize db access --- src/Caching/Impl/ModuleDatabase.cs | 31 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index ed2670534..d36ba4440 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -33,10 +33,14 @@ namespace Microsoft.Python.Analysis.Caching { internal sealed class ModuleDatabase : IModuleDatabaseService { - private readonly object _lock = new object(); - private static readonly Dictionary _modulesCache + private readonly object _modulesLock = new object(); + private readonly Dictionary _modulesCache = new Dictionary(); - private static readonly ConcurrentDictionary _searchResults + + private readonly ConcurrentDictionary _modelsCache + = new ConcurrentDictionary(); + + private readonly ConcurrentDictionary _searchResults = new ConcurrentDictionary(); private readonly IServiceContainer _services; @@ -103,7 +107,7 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul } internal IPythonModule RestoreModule(string moduleName, string uniqueId) { - lock (_lock) { + lock (_modulesLock) { if (_modulesCache.TryGetValue(uniqueId, out var m)) { return m; } @@ -113,7 +117,7 @@ internal IPythonModule RestoreModule(string moduleName, string uniqueId) { private IPythonModule RestoreModule(ModuleModel model) { PythonDbModule dbModule; - lock (_lock) { + lock (_modulesLock) { if (_modulesCache.TryGetValue(model.UniqueId, out var m)) { return m; } @@ -147,6 +151,7 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c using (var db = new LiteDatabase(Path.Combine(CacheFolder, $"{model.UniqueId}.db"))) { var modules = db.GetCollection("modules"); modules.Upsert(model); + modules.EnsureIndex(x => x.Name); } return true; }, $"Unable to write analysis of {model.Name} to database."); @@ -192,18 +197,24 @@ private bool FindModuleModelById(string moduleName, string uniqueId, out ModuleM private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel model) { model = null; - + if (string.IsNullOrEmpty(dbPath)) { return false; } - - model = WithRetries(() => { + + if (_modelsCache.TryGetValue(moduleName, out model)) { + return true; + } + + model = WithRetries(() => { using (var db = new LiteDatabase(dbPath)) { var modules = db.GetCollection("modules"); - return modules.Find(m => m.Name == moduleName).FirstOrDefault(); + var storedModel = modules.FindOne(m => m.Name == moduleName); + _modelsCache[moduleName] = storedModel; + return storedModel; } }, $"Unable to locate database for module {moduleName}."); - + return model != null; } From d7a6feaa4c3283cce357e74bde99aad63dec07af Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 09:20:44 -0800 Subject: [PATCH 039/102] fix test --- src/Core/Impl/IO/PathUtils.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Core/Impl/IO/PathUtils.cs b/src/Core/Impl/IO/PathUtils.cs index 3450a03f8..6029cc30a 100644 --- a/src/Core/Impl/IO/PathUtils.cs +++ b/src/Core/Impl/IO/PathUtils.cs @@ -286,6 +286,9 @@ public static IEnumerable EnumerateFiles(IFileSystem fileSystem, stri foreach (var dir in dirs) { var fullDir = Path.IsPathRooted(dir) ? dir : root + dir; + if (string.IsNullOrEmpty(fullDir)) { + continue; + } IFileInfo[] files = null; try { files = fileSystem.GetDirectoryInfo(fullDir) From f109701710adba86245c64e9070a4c4f0d21e15c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 09:31:09 -0800 Subject: [PATCH 040/102] Remove debug code --- src/LanguageServer/Impl/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 85abd38b6..25528cdb5 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From c4891cfe8ed2b952d50d9f1dc6c3c9f7b36aeddd Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 09:55:50 -0800 Subject: [PATCH 041/102] Fix comment --- src/Caching/Impl/ModuleFactory.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index ac53df9a0..28e6a94b2 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -152,9 +152,7 @@ private IPythonModule GetModule(QualifiedNameParts parts) { if (module == null) { // Fallback if somehow module is not loaded or missing from the database. - // Try to load it directly and wait a bit hoping for the analysis to complete. - // Example: posix is a stub-only and is not loaded with the database. - // TODO: consider writing stub-only modules to the database as well. + // Try loading it directly and wait a bit hoping for the analysis to complete. var resolution = parts.IsStub ? tres : mres; var imports = resolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, Enumerable.Repeat(parts.ModuleName, 1), true); if (imports is ModuleImport moduleImport) { From 6726924955b436ef31c6f5f0bc341cc922e2cf17 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 09:57:41 -0800 Subject: [PATCH 042/102] Delete debug file --- src/json1.json | 7980 ------------------------------------------------ 1 file changed, 7980 deletions(-) delete mode 100644 src/json1.json diff --git a/src/json1.json b/src/json1.json deleted file mode 100644 index 91e0808c8..000000000 --- a/src/json1.json +++ /dev/null @@ -1,7980 +0,0 @@ -{ - "UniqueId": "ctypes(3.7)", - "FilePath": "c:\\users\\mikhaila\\appdata\\local\\programs\\python\\python37\\lib\\ctypes\\__init__.py", - "Documentation": "create and manipulate C data types in Python", - "Functions": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "init_or_size", - "Type": "t:typing:Union[int, bytes]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "size", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):Array[ctypes(stub):c_char]$ctypes(3.7)", - "Documentation": "create_string_buffer(aBytes) -> character array\n create_string_buffer(anInteger) -> character array\n create_string_buffer(aBytes, anInteger) -> character array\n " - } - ], - "Documentation": "create_string_buffer(aBytes) -> character array\n create_string_buffer(anInteger) -> character array\n create_string_buffer(aBytes, anInteger) -> character array\n ", - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -709606206, - "Name": "create_string_buffer", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:create_string_buffer", - "IndexSpan": { - "Start": 1304, - "Length": 20 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "restype", - "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "argtypes", - "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", - "DefaultValue": null, - "Kind": 1 - }, - { - "Name": "use_errno", - "Type": "t:bool", - "DefaultValue": "i:...", - "Kind": 3 - }, - { - "Name": "use_last_error", - "Type": "t:bool", - "DefaultValue": "i:...", - "Kind": 3 - } - ], - "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Documentation": "CFUNCTYPE(restype, *argtypes,\n use_errno=False, use_last_error=False) -> function prototype.\n\n restype: the result type\n argtypes: a sequence specifying the argument types\n\n The function prototype can be called in different ways to create a\n callable object:\n\n prototype(integer address) -> foreign function\n prototype(callable) -> create and return a C callable function from callable\n prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method\n prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal\n prototype((function name, dll object)[, paramflags]) -> foreign function exported by name\n " - } - ], - "Documentation": "CFUNCTYPE(restype, *argtypes,\n use_errno=False, use_last_error=False) -> function prototype.\n\n restype: the result type\n argtypes: a sequence specifying the argument types\n\n The function prototype can be called in different ways to create a\n callable object:\n\n prototype(integer address) -> foreign function\n prototype(callable) -> create and return a C callable function from callable\n prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method\n prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal\n prototype((function name, dll object)[, paramflags]) -> foreign function exported by name\n ", - "Attributes": 0, - "Classes": [ - { - "Documentation": null, - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": -1921867084, - "Name": "_argtypes_", - "DeclaringModuleId": null, - "QualifiedName": "_argtypes_", - "IndexSpan": null - }, - { - "Value": null, - "Id": -470655251, - "Name": "_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_restype_", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1428625440, - "Name": "_flags_", - "DeclaringModuleId": null, - "QualifiedName": "_flags_", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1267928994, - "Name": "CFunctionType", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:CFUNCTYPE.CFunctionType", - "IndexSpan": { - "Start": 3342, - "Length": 13 - } - } - ], - "Functions": [], - "Id": 1098372874, - "Name": "CFUNCTYPE", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:CFUNCTYPE", - "IndexSpan": { - "Start": 2184, - "Length": 9 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "restype", - "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "argtypes", - "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", - "DefaultValue": null, - "Kind": 1 - }, - { - "Name": "use_errno", - "Type": "t:bool", - "DefaultValue": "i:...", - "Kind": 3 - }, - { - "Name": "use_last_error", - "Type": "t:bool", - "DefaultValue": "i:...", - "Kind": 3 - } - ], - "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [ - { - "Documentation": null, - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": -1921867084, - "Name": "_argtypes_", - "DeclaringModuleId": null, - "QualifiedName": "_argtypes_", - "IndexSpan": null - }, - { - "Value": null, - "Id": -470655251, - "Name": "_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_restype_", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1428625440, - "Name": "_flags_", - "DeclaringModuleId": null, - "QualifiedName": "_flags_", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -349853673, - "Name": "WinFunctionType", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:WINFUNCTYPE.WinFunctionType", - "IndexSpan": { - "Start": 4278, - "Length": 15 - } - } - ], - "Functions": [], - "Id": -1586540733, - "Name": "WINFUNCTYPE", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:WINFUNCTYPE", - "IndexSpan": { - "Start": 3742, - "Length": 11 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "typ", - "Type": null, - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "typecode", - "Type": null, - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -756102656, - "Name": "_check_size", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:_check_size", - "IndexSpan": { - "Start": 4866, - "Length": 11 - } - }, - { - "Overloads": [ - { - "Parameters": [], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 774351978, - "Name": "_reset_cache", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:_reset_cache", - "IndexSpan": { - "Start": 8078, - "Length": 12 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "init_or_size", - "Type": "t:typing:Union[int, Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "size", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):Array[ctypes(stub):c_wchar]$ctypes(3.7)", - "Documentation": "create_unicode_buffer(aString) -> character array\n create_unicode_buffer(anInteger) -> character array\n create_unicode_buffer(aString, anInteger) -> character array\n " - } - ], - "Documentation": "create_unicode_buffer(aString) -> character array\n create_unicode_buffer(anInteger) -> character array\n create_unicode_buffer(aString, anInteger) -> character array\n ", - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 861572558, - "Name": "create_unicode_buffer", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:create_unicode_buffer", - "IndexSpan": { - "Start": 8792, - "Length": 21 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "pointer", - "Type": null, - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "cls", - "Type": null, - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1963781452, - "Name": "SetPointerType", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:SetPointerType", - "IndexSpan": { - "Start": 9382, - "Length": 14 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "typ", - "Type": null, - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "len", - "Type": null, - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 721023106, - "Name": "ARRAY", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:ARRAY", - "IndexSpan": { - "Start": 9765, - "Length": 5 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "code", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - }, - { - "Name": "desc", - "Type": "t:str", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:OSError", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 298941635, - "Name": "WinError", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:WinError", - "IndexSpan": { - "Start": 14822, - "Length": 8 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "restype", - "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "argtypes", - "Type": "t:typing:Type[ctypes(stub):_CData]$typing(3.7)", - "DefaultValue": null, - "Kind": 1 - } - ], - "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [ - { - "Documentation": null, - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": -1921867084, - "Name": "_argtypes_", - "DeclaringModuleId": null, - "QualifiedName": "_argtypes_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": -470655251, - "Name": "_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_restype_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1267928994, - "Name": "CFunctionType", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:PYFUNCTYPE.CFunctionType", - "IndexSpan": { - "Start": 15704, - "Length": 13 - } - } - ], - "Functions": [], - "Id": -1002463138, - "Name": "PYFUNCTYPE", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:PYFUNCTYPE", - "IndexSpan": { - "Start": 15661, - "Length": 10 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "obj", - "Type": "t:typing:Union[ctypes(stub):_CData, ctypes(stub):_CArgObject]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "type", - "Type": "t:typing:Type[ctypes:_PT]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes:_PT$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 24287190, - "Name": "cast", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:cast", - "IndexSpan": { - "Start": 15955, - "Length": 4 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "address", - "Type": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "size", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:bytes", - "Documentation": "string_at(addr[, size]) -> string\n\n Return the string at addr." - } - ], - "Documentation": "string_at(addr[, size]) -> string\n\n Return the string at addr.", - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -414124822, - "Name": "string_at", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:string_at", - "IndexSpan": { - "Start": 16081, - "Length": 9 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "address", - "Type": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "size", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:str", - "Documentation": "wstring_at(addr[, size]) -> string\n\n Return the string at addr." - } - ], - "Documentation": "wstring_at(addr[, size]) -> string\n\n Return the string at addr.", - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -479760415, - "Name": "wstring_at", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:wstring_at", - "IndexSpan": { - "Start": 16392, - "Length": 10 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "rclsid", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "riid", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "ppv", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -1849292722, - "Name": "DllGetClassObject", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:DllGetClassObject", - "IndexSpan": { - "Start": 16592, - "Length": 17 - } - }, - { - "Overloads": [ - { - "Parameters": [], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1652175700, - "Name": "DllCanUnloadNow", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:DllCanUnloadNow", - "IndexSpan": { - "Start": 16912, - "Length": 15 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "code", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:str", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1776340922, - "Name": "FormatError", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):FormatError", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "obj_or_type", - "Type": "t:typing:Union[ctypes(stub):_CData, typing:Type[ctypes(stub):_CData]]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -1964161937, - "Name": "sizeof", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):sizeof", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "obj", - "Type": "t:ctypes(stub):_CData$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "offset", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):_CArgObject$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 752693029, - "Name": "byref", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):byref", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "obj", - "Type": "t:ctypes(stub):_CData$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 649701012, - "Name": "addressof", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):addressof", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "obj_or_type", - "Type": "t:typing:Union[ctypes(stub):_CData, typing:Type[ctypes(stub):_CData]]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1543031724, - "Name": "alignment", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):alignment", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "obj", - "Type": "t:ctypes(stub):_CData$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "size", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -1996689525, - "Name": "resize", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):resize", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 908891526, - "Name": "get_errno", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):get_errno", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "value", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 694274450, - "Name": "set_errno", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):set_errno", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "type", - "Type": "t:typing:Type[ctypes:_CT]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1747309094, - "Name": "POINTER", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):POINTER", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1833420806, - "Name": "_check_HRESULT", - "DeclaringModuleId": "_ctypes(3.7)", - "QualifiedName": "_ctypes:_check_HRESULT", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -309340141, - "Name": "GetLastError", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):GetLastError", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1363574655, - "Name": "get_last_error", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):get_last_error", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "value", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -408161549, - "Name": "set_last_error", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):set_last_error", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "dst", - "Type": "t:typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "src", - "Type": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "count", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -1915816145, - "Name": "memmove", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):memmove", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "dst", - "Type": "t:typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "c", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "count", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -2140005052, - "Name": "memset", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):memset", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Variables": [ - { - "Value": "t:ctypes:create_string_buffer$ctypes(3.7)", - "Id": -846863181, - "Name": "c_buffer", - "DeclaringModuleId": null, - "QualifiedName": "c_buffer", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "t:ctypes:c_void_p$ctypes(3.7)", - "Id": 1930638985, - "Name": "c_voidp", - "DeclaringModuleId": null, - "QualifiedName": "c_voidp", - "IndexSpan": { - "Start": 7668, - "Length": 7 - } - }, - { - "Value": "p:os$os(3.7)", - "Id": 780044, - "Name": "_os", - "DeclaringModuleId": null, - "QualifiedName": "_os", - "IndexSpan": { - "Start": 67, - "Length": 3 - } - }, - { - "Value": "p:sys$sys(3.7)", - "Id": 24185509, - "Name": "_sys", - "DeclaringModuleId": null, - "QualifiedName": "_sys", - "IndexSpan": { - "Start": 79, - "Length": 4 - } - }, - { - "Value": "i:str", - "Id": 1161199201, - "Name": "__version__", - "DeclaringModuleId": null, - "QualifiedName": "__version__", - "IndexSpan": { - "Start": 87, - "Length": 11 - } - }, - { - "Value": "t:_ctypes:_Pointer$_ctypes(3.7)", - "Id": 705525237, - "Name": "_Pointer", - "DeclaringModuleId": null, - "QualifiedName": "_Pointer", - "IndexSpan": { - "Start": 177, - "Length": 8 - } - }, - { - "Value": null, - "Id": 1824826095, - "Name": "_CFuncPtr", - "DeclaringModuleId": null, - "QualifiedName": "_CFuncPtr", - "IndexSpan": { - "Start": 219, - "Length": 9 - } - }, - { - "Value": "i:str", - "Id": 1827465399, - "Name": "_ctypes_version", - "DeclaringModuleId": null, - "QualifiedName": "_ctypes_version", - "IndexSpan": { - "Start": 265, - "Length": 15 - } - }, - { - "Value": "i:int", - "Id": -789048995, - "Name": "RTLD_LOCAL", - "DeclaringModuleId": null, - "QualifiedName": "RTLD_LOCAL", - "IndexSpan": { - "Start": 302, - "Length": 10 - } - }, - { - "Value": "i:int", - "Id": 1163726801, - "Name": "RTLD_GLOBAL", - "DeclaringModuleId": null, - "QualifiedName": "RTLD_GLOBAL", - "IndexSpan": { - "Start": 314, - "Length": 11 - } - }, - { - "Value": "t:struct(stub):calcsize$struct(3.7)", - "Id": -314342658, - "Name": "_calcsize", - "DeclaringModuleId": null, - "QualifiedName": "_calcsize", - "IndexSpan": { - "Start": 395, - "Length": 9 - } - }, - { - "Value": "i:int", - "Id": -756921704, - "Name": "DEFAULT_MODE", - "DeclaringModuleId": null, - "QualifiedName": "DEFAULT_MODE", - "IndexSpan": { - "Start": 585, - "Length": 12 - } - }, - { - "Value": "i:int", - "Id": 1661077446, - "Name": "_FUNCFLAG_CDECL", - "DeclaringModuleId": null, - "QualifiedName": "_FUNCFLAG_CDECL", - "IndexSpan": { - "Start": 981, - "Length": 15 - } - }, - { - "Value": "i:int", - "Id": -1361874057, - "Name": "_FUNCFLAG_PYTHONAPI", - "DeclaringModuleId": null, - "QualifiedName": "_FUNCFLAG_PYTHONAPI", - "IndexSpan": { - "Start": 1028, - "Length": 19 - } - }, - { - "Value": "i:int", - "Id": 840545479, - "Name": "_FUNCFLAG_USE_ERRNO", - "DeclaringModuleId": null, - "QualifiedName": "_FUNCFLAG_USE_ERRNO", - "IndexSpan": { - "Start": 1079, - "Length": 19 - } - }, - { - "Value": "i:int", - "Id": -675549293, - "Name": "_FUNCFLAG_USE_LASTERROR", - "DeclaringModuleId": null, - "QualifiedName": "_FUNCFLAG_USE_LASTERROR", - "IndexSpan": { - "Start": 1134, - "Length": 23 - } - }, - { - "Value": "i:dict$ctypes(3.7)", - "Id": -843695483, - "Name": "_c_functype_cache", - "DeclaringModuleId": null, - "QualifiedName": "_c_functype_cache", - "IndexSpan": { - "Start": 2156, - "Length": 17 - } - }, - { - "Value": "t:_ctypes:LoadLibrary$_ctypes(3.7)", - "Id": -1485458438, - "Name": "_dlopen", - "DeclaringModuleId": null, - "QualifiedName": "_dlopen", - "IndexSpan": { - "Start": 3630, - "Length": 7 - } - }, - { - "Value": "i:int", - "Id": 339883258, - "Name": "_FUNCFLAG_STDCALL", - "DeclaringModuleId": null, - "QualifiedName": "_FUNCFLAG_STDCALL", - "IndexSpan": { - "Start": 3683, - "Length": 17 - } - }, - { - "Value": "i:dict$ctypes(3.7)", - "Id": 2115421164, - "Name": "_win_functype_cache", - "DeclaringModuleId": null, - "QualifiedName": "_win_functype_cache", - "IndexSpan": { - "Start": 3708, - "Length": 19 - } - }, - { - "Value": "i:dict", - "Id": 456857255, - "Name": "_pointer_type_cache", - "DeclaringModuleId": null, - "QualifiedName": "_pointer_type_cache", - "IndexSpan": { - "Start": 7837, - "Length": 19 - } - }, - { - "Value": "i:ctypes:LibraryLoader[ctypes(stub):CDLL]$ctypes(3.7)", - "Id": 24289848, - "Name": "cdll", - "DeclaringModuleId": null, - "QualifiedName": "cdll", - "IndexSpan": { - "Start": 14328, - "Length": 4 - } - }, - { - "Value": "i:ctypes:LibraryLoader[ctypes(stub):PyDLL]$ctypes(3.7)", - "Id": 765609092, - "Name": "pydll", - "DeclaringModuleId": null, - "QualifiedName": "pydll", - "IndexSpan": { - "Start": 14356, - "Length": 5 - } - }, - { - "Value": "i:ctypes:PyDLL$ctypes(3.7)", - "Id": -1710066297, - "Name": "pythonapi", - "DeclaringModuleId": null, - "QualifiedName": "pythonapi", - "IndexSpan": { - "Start": 14414, - "Length": 9 - } - }, - { - "Value": "i:ctypes:LibraryLoader[ctypes(stub):WinDLL]$ctypes(3.7)", - "Id": -1850003873, - "Name": "windll", - "DeclaringModuleId": null, - "QualifiedName": "windll", - "IndexSpan": { - "Start": 14637, - "Length": 6 - } - }, - { - "Value": "i:ctypes:LibraryLoader[ctypes(stub):OleDLL]$ctypes(3.7)", - "Id": -2076534637, - "Name": "oledll", - "DeclaringModuleId": null, - "QualifiedName": "oledll", - "IndexSpan": { - "Start": 14673, - "Length": 6 - } - }, - { - "Value": "i:int", - "Id": 357987474, - "Name": "_memmove_addr", - "DeclaringModuleId": null, - "QualifiedName": "_memmove_addr", - "IndexSpan": { - "Start": 15359, - "Length": 13 - } - }, - { - "Value": "i:int", - "Id": -2071594405, - "Name": "_memset_addr", - "DeclaringModuleId": null, - "QualifiedName": "_memset_addr", - "IndexSpan": { - "Start": 15374, - "Length": 12 - } - }, - { - "Value": "i:int", - "Id": -1481723017, - "Name": "_string_at_addr", - "DeclaringModuleId": null, - "QualifiedName": "_string_at_addr", - "IndexSpan": { - "Start": 15388, - "Length": 15 - } - }, - { - "Value": "i:int", - "Id": 1230836425, - "Name": "_cast_addr", - "DeclaringModuleId": null, - "QualifiedName": "_cast_addr", - "IndexSpan": { - "Start": 15405, - "Length": 10 - } - }, - { - "Value": "i:typing:Any$typing(3.7)", - "Id": 749251175, - "Name": "_cast", - "DeclaringModuleId": null, - "QualifiedName": "_cast", - "IndexSpan": { - "Start": 15876, - "Length": 5 - } - }, - { - "Value": "i:typing:Any$typing(3.7)", - "Id": -58403591, - "Name": "_string_at", - "DeclaringModuleId": null, - "QualifiedName": "_string_at", - "IndexSpan": { - "Start": 16007, - "Length": 10 - } - }, - { - "Value": "i:int", - "Id": -997588386, - "Name": "_wstring_at_addr", - "DeclaringModuleId": null, - "QualifiedName": "_wstring_at_addr", - "IndexSpan": { - "Start": 16252, - "Length": 16 - } - }, - { - "Value": "i:typing:Any$typing(3.7)", - "Id": 1957663154, - "Name": "_wstring_at", - "DeclaringModuleId": null, - "QualifiedName": "_wstring_at", - "IndexSpan": { - "Start": 16312, - "Length": 11 - } - }, - { - "Value": "t:ctypes._endian:BigEndianStructure$ctypes._endian(3.7)", - "Id": 2047389551, - "Name": "BigEndianStructure", - "DeclaringModuleId": null, - "QualifiedName": "BigEndianStructure", - "IndexSpan": { - "Start": 17163, - "Length": 18 - } - }, - { - "Value": "t:ctypes:c_ushort$ctypes(3.7)", - "Id": 24533035, - "Name": "kind", - "DeclaringModuleId": null, - "QualifiedName": "kind", - "IndexSpan": { - "Start": 17284, - "Length": 4 - } - }, - { - "Value": "t:typing:Union[typing:Tuple[int], typing:Tuple[int, str], typing:Tuple[int, str, typing:Any]]$typing(3.7)", - "Id": 779038, - "Name": "_PF", - "DeclaringModuleId": null, - "QualifiedName": "_PF", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "t:typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int]$typing(3.7)", - "Id": 474673944, - "Name": "_CVoidPLike", - "DeclaringModuleId": null, - "QualifiedName": "_CVoidPLike", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "t:typing:Union[typing:Union[ctypes(stub):_PointerLike, ctypes(stub):Array[typing:Any], ctypes(stub):_CArgObject, int], bytes]$typing(3.7)", - "Id": 2128974387, - "Name": "_CVoidConstPLike", - "DeclaringModuleId": null, - "QualifiedName": "_CVoidConstPLike", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Classes": [ - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[ctypes:_T]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:ctypes:_T$ctypes(3.7)", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1315475454, - "Name": "py_object", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:py_object", - "IndexSpan": { - "Start": 5392, - "Length": 9 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1927666089, - "Name": "c_short", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_short", - "IndexSpan": { - "Start": 5634, - "Length": 7 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -304687718, - "Name": "c_ushort", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_ushort", - "IndexSpan": { - "Start": 5706, - "Length": 8 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1863096239, - "Name": "c_long", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_long", - "IndexSpan": { - "Start": 5780, - "Length": 6 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1929632158, - "Name": "c_ulong", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_ulong", - "IndexSpan": { - "Start": 5850, - "Length": 7 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 752833628, - "Name": "c_int", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_int", - "IndexSpan": { - "Start": 6087, - "Length": 5 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1863358605, - "Name": "c_uint", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_uint", - "IndexSpan": { - "Start": 6167, - "Length": 6 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[float]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:float", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1915778953, - "Name": "c_float", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_float", - "IndexSpan": { - "Start": 6245, - "Length": 7 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[float]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:float", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -794702780, - "Name": "c_double", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_double", - "IndexSpan": { - "Start": 6317, - "Length": 8 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[float]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:float", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1655090112, - "Name": "c_longdouble", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_longdouble", - "IndexSpan": { - "Start": 6317, - "Length": 8 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1656614571, - "Name": "c_longlong", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_longlong", - "IndexSpan": { - "Start": 6707, - "Length": 10 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 877961498, - "Name": "c_ulonglong", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_ulonglong", - "IndexSpan": { - "Start": 6797, - "Length": 11 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1929344042, - "Name": "c_ubyte", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_ubyte", - "IndexSpan": { - "Start": 7015, - "Length": 7 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1862808123, - "Name": "c_byte", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_byte", - "IndexSpan": { - "Start": 7190, - "Length": 6 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[bytes]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:c_char$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "value", - "Type": "t:typing:Union[int, bytes]$typing(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):c_char.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:bytes", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1862821001, - "Name": "c_char", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_char", - "IndexSpan": { - "Start": 7312, - "Length": 6 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_PointerLike$ctypes(3.7)", - "t:ctypes(stub):_SimpleCData[bytes]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:c_char_p$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "value", - "Type": "t:typing:Union[int, bytes]$typing(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):c_char_p.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - }, - { - "Value": "i:bytes", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -830377414, - "Name": "c_char_p", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_char_p", - "IndexSpan": { - "Start": 7434, - "Length": 8 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_PointerLike$ctypes(3.7)", - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -279734024, - "Name": "c_void_p", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_void_p", - "IndexSpan": { - "Start": 7625, - "Length": 8 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[bool]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:c_bool$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "value", - "Type": "t:bool", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):c_bool.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:bool", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1862798365, - "Name": "c_bool", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_bool", - "IndexSpan": { - "Start": 7756, - "Length": 6 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_PointerLike$ctypes(3.7)", - "t:ctypes(stub):_SimpleCData[ctypes:_T]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:c_wchar_p$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "value", - "Type": "t:typing:Union[int, Unknown]$typing(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):c_wchar_p.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - }, - { - "Value": "i:ctypes:_T$ctypes(3.7)", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 461138667, - "Name": "c_wchar_p", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_wchar_p", - "IndexSpan": { - "Start": 7866, - "Length": 9 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[ctypes:_T]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:ctypes:_T$ctypes(3.7)", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1931203962, - "Name": "c_wchar", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_wchar", - "IndexSpan": { - "Start": 8030, - "Length": 7 - } - }, - { - "Documentation": "An instance of this class represents a loaded dll/shared\n library, exporting functions using the standard C calling\n convention (named 'cdecl' on Windows).\n\n The exported functions can be accessed as attributes, or by\n indexing with the function name. Examples:\n\n .qsort -> callable object\n ['qsort'] -> callable object\n\n Calling the functions releases the Python GIL during the call and\n reacquires it afterwards.\n ", - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:CDLL$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "mode", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - }, - { - "Name": "handle", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - }, - { - "Name": "use_errno", - "Type": "t:bool", - "DefaultValue": "i:...", - "Kind": 0 - }, - { - "Name": "use_last_error", - "Type": "t:bool", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": "An instance of this class represents a loaded dll/shared\n library, exporting functions using the standard C calling\n convention (named 'cdecl' on Windows).\n\n The exported functions can be accessed as attributes, or by\n indexing with the function name. Examples:\n\n .qsort -> callable object\n ['qsort'] -> callable object\n\n Calling the functions releases the Python GIL during the call and\n reacquires it afterwards.\n " - } - ], - "Documentation": "An instance of this class represents a loaded dll/shared\n library, exporting functions using the standard C calling\n convention (named 'cdecl' on Windows).\n\n The exported functions can be accessed as attributes, or by\n indexing with the function name. Examples:\n\n .qsort -> callable object\n ['qsort'] -> callable object\n\n Calling the functions releases the Python GIL during the call and\n reacquires it afterwards.\n ", - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:CDLL.__init__", - "IndexSpan": { - "Start": 10550, - "Length": 8 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:CDLL$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -521760752, - "Name": "__getattr__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:CDLL.__getattr__", - "IndexSpan": { - "Start": 11882, - "Length": 11 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:CDLL$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -293179214, - "Name": "__getitem__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:CDLL.__getitem__", - "IndexSpan": { - "Start": 12112, - "Length": 11 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": 2021379179, - "Name": "_func_flags_", - "DeclaringModuleId": null, - "QualifiedName": "_func_flags_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": -730968904, - "Name": "_func_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_func_restype_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:str", - "Id": 749578675, - "Name": "_name", - "DeclaringModuleId": null, - "QualifiedName": "_name", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1381141680, - "Name": "_handle", - "DeclaringModuleId": null, - "QualifiedName": "_handle", - "IndexSpan": null - }, - { - "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Id": 596453698, - "Name": "_FuncPtr", - "DeclaringModuleId": null, - "QualifiedName": "_FuncPtr", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 23304760, - "Name": "CDLL", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:CDLL", - "IndexSpan": { - "Start": 9883, - "Length": 4 - } - }, - { - "Documentation": "This class represents the Python library itself. It allows\n accessing Python API functions. The GIL is not released, and\n Python exceptions are handled correctly.\n ", - "Bases": [ - "t:ctypes:CDLL$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": 2021379179, - "Name": "_func_flags_", - "DeclaringModuleId": null, - "QualifiedName": "_func_flags_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": -730968904, - "Name": "_func_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_func_restype_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:str", - "Id": 749578675, - "Name": "_name", - "DeclaringModuleId": null, - "QualifiedName": "_name", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1381141680, - "Name": "_handle", - "DeclaringModuleId": null, - "QualifiedName": "_handle", - "IndexSpan": null - }, - { - "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Id": 596453698, - "Name": "_FuncPtr", - "DeclaringModuleId": null, - "QualifiedName": "_FuncPtr", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 736024644, - "Name": "PyDLL", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:PyDLL", - "IndexSpan": { - "Start": 12328, - "Length": 5 - } - }, - { - "Documentation": "This class represents a dll exporting functions using the\n Windows stdcall calling convention.\n ", - "Bases": [ - "t:ctypes:CDLL$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": 2021379179, - "Name": "_func_flags_", - "DeclaringModuleId": null, - "QualifiedName": "_func_flags_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": -730968904, - "Name": "_func_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_func_restype_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:str", - "Id": 749578675, - "Name": "_name", - "DeclaringModuleId": null, - "QualifiedName": "_name", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1381141680, - "Name": "_handle", - "DeclaringModuleId": null, - "QualifiedName": "_handle", - "IndexSpan": null - }, - { - "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Id": 596453698, - "Name": "_FuncPtr", - "DeclaringModuleId": null, - "QualifiedName": "_FuncPtr", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1528798815, - "Name": "WinDLL", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:WinDLL", - "IndexSpan": { - "Start": 12626, - "Length": 6 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -975409554, - "Name": "HRESULT", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:HRESULT", - "IndexSpan": { - "Start": 12975, - "Length": 7 - } - }, - { - "Documentation": "This class represents a dll exporting functions using the\n Windows stdcall calling convention, and returning HRESULT.\n HRESULT error values are automatically raised as OSError\n exceptions.\n ", - "Bases": [ - "t:ctypes:CDLL$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": 2021379179, - "Name": "_func_flags_", - "DeclaringModuleId": null, - "QualifiedName": "_func_flags_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": -730968904, - "Name": "_func_restype_", - "DeclaringModuleId": null, - "QualifiedName": "_func_restype_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:str", - "Id": 749578675, - "Name": "_name", - "DeclaringModuleId": null, - "QualifiedName": "_name", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1381141680, - "Name": "_handle", - "DeclaringModuleId": null, - "QualifiedName": "_handle", - "IndexSpan": null - }, - { - "Value": "i:ctypes(stub):_FuncPointer$ctypes(3.7)", - "Id": 596453698, - "Name": "_FuncPtr", - "DeclaringModuleId": null, - "QualifiedName": "_FuncPtr", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1302268051, - "Name": "OleDLL", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:OleDLL", - "IndexSpan": { - "Start": 13561, - "Length": 6 - } - }, - { - "Documentation": null, - "Bases": [ - "t:typing:Generic$typing(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "dlltype", - "Type": "t:typing:Type[ctypes:_DLLT]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:LibraryLoader.__init__", - "IndexSpan": { - "Start": 13930, - "Length": 8 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes:_DLLT$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -521760752, - "Name": "__getattr__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:LibraryLoader.__getattr__", - "IndexSpan": { - "Start": 13999, - "Length": 11 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes:_DLLT$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -293179214, - "Name": "__getitem__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:LibraryLoader.__getitem__", - "IndexSpan": { - "Start": 14192, - "Length": 11 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes:LibraryLoader$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes:_DLLT$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -4764578, - "Name": "LoadLibrary", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:LibraryLoader.LoadLibrary", - "IndexSpan": { - "Start": 14264, - "Length": 11 - } - } - ], - "Properties": [], - "Fields": [], - "Classes": [], - "GenericBaseParameters": [ - "_DLLT" - ], - "GenericParameterValues": [], - "Id": 49337815, - "Name": "LibraryLoader", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:LibraryLoader", - "IndexSpan": { - "Start": 13898, - "Length": 13 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1863006044, - "Name": "c_int8", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_int8", - "IndexSpan": { - "Start": 7190, - "Length": 6 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1929541963, - "Name": "c_uint8", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_uint8", - "IndexSpan": { - "Start": 7015, - "Length": 7 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 740351224, - "Name": "Union", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Union", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1923643164, - "Name": "Structure", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Structure", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:typing:Generic$typing(3.7)", - "t:ctypes(stub):_CData$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "args", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 1 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Array.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "i", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:typing:Any$typing(3.7)", - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "s", - "Type": "t:slice", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:typing:List[typing:Any]$typing(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -293179214, - "Name": "__getitem__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Array.__getitem__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "i", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "o", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "s", - "Type": "t:slice", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "o", - "Type": "t:typing:Iterable[typing:Any]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -507796290, - "Name": "__setitem__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Array.__setitem__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:list_iterator", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 971292143, - "Name": "__iter__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Array.__iter__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):Array$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:int", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -1628904226, - "Name": "__len__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Array.__len__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": -496057839, - "Name": "_length_", - "DeclaringModuleId": null, - "QualifiedName": "_length_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": 1768361533, - "Name": "_type_", - "DeclaringModuleId": null, - "QualifiedName": "_type_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:bytes", - "Id": 797873, - "Name": "raw", - "DeclaringModuleId": null, - "QualifiedName": "raw", - "IndexSpan": null - }, - { - "Value": "i:bytes", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [ - "_T" - ], - "GenericParameterValues": [], - "Id": 722008194, - "Name": "Array", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):Array", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": "Common base class for all non-exit exceptions.", - "Bases": [ - "t:Exception", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1992130164, - "Name": "ArgumentError", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):ArgumentError", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:typing:Generic$typing(3.7)", - "t:ctypes(stub):_CData$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_SimpleCData$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "value", - "Type": "t:ctypes:_T$ctypes(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_SimpleCData.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:ctypes:_T$ctypes(3.7)", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [ - "_T" - ], - "GenericParameterValues": [], - "Id": 1441346515, - "Name": "_SimpleCData", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_SimpleCData", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_PointerLike$ctypes(3.7)", - "t:ctypes(stub):_CData$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):pointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "arg", - "Type": "t:ellipsis", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):pointer.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):pointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "i", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):pointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "s", - "Type": "t:slice", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:typing:List[Unknown]$typing(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -293179214, - "Name": "__getitem__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):pointer.__getitem__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):pointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "i", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "o", - "Type": null, - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):pointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "s", - "Type": "t:slice", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "o", - "Type": "t:typing:Iterable[Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -507796290, - "Name": "__setitem__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):pointer.__setitem__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": 1768361533, - "Name": "_type_", - "DeclaringModuleId": null, - "QualifiedName": "_type_", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": null, - "Id": 810898481, - "Name": "contents", - "DeclaringModuleId": null, - "QualifiedName": "contents", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1029326406, - "Name": "pointer", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):pointer", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -370655191, - "Name": "c_size_t", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_size_t", - "IndexSpan": { - "Start": 6797, - "Length": 11 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1665813462, - "Name": "c_ssize_t", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_ssize_t", - "IndexSpan": { - "Start": 6707, - "Length": 10 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):Structure$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1726755221, - "Name": "LittleEndianStructure", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):LittleEndianStructure", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1918612353, - "Name": "c_int16", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_int16", - "IndexSpan": { - "Start": 5634, - "Length": 7 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1918612411, - "Name": "c_int32", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_int32", - "IndexSpan": { - "Start": 5634, - "Length": 7 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1918612506, - "Name": "c_int64", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_int64", - "IndexSpan": { - "Start": 5634, - "Length": 7 - } - }, - { - "Documentation": "XXX to be provided", - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -313741454, - "Name": "c_uint16", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_uint16", - "IndexSpan": { - "Start": 5706, - "Length": 8 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -313741396, - "Name": "c_uint32", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_uint32", - "IndexSpan": { - "Start": 5706, - "Length": 8 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_SimpleCData[int]$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": 770443194, - "Name": "value", - "DeclaringModuleId": null, - "QualifiedName": "value", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -313741301, - "Name": "c_uint64", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes:c_uint64", - "IndexSpan": { - "Start": 5706, - "Length": 8 - } - }, - { - "Documentation": "type(object_or_name, bases, dict)\ntype(object) -> the object's type\ntype(name, bases, dict) -> a new type", - "Bases": [ - "t:type", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[ctypes:_CT]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "other", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):Array[ctypes:_CT]$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -1627505971, - "Name": "__mul__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CDataMeta.__mul__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[ctypes:_CT]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "other", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):Array[ctypes:_CT]$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 1222960745, - "Name": "__rmul__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CDataMeta.__rmul__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": null, - "Id": -1388753224, - "Name": "__basicsize__", - "DeclaringModuleId": null, - "QualifiedName": "__basicsize__", - "IndexSpan": null - }, - { - "Value": null, - "Id": -198224608, - "Name": "__dictoffset__", - "DeclaringModuleId": null, - "QualifiedName": "__dictoffset__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 1444705936, - "Name": "__flags__", - "DeclaringModuleId": null, - "QualifiedName": "__flags__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 919460331, - "Name": "__itemsize__", - "DeclaringModuleId": null, - "QualifiedName": "__itemsize__", - "IndexSpan": null - }, - { - "Value": "i:list$ctypes(3.7)", - "Id": -1627592461, - "Name": "__mro__", - "DeclaringModuleId": null, - "QualifiedName": "__mro__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 1097116834, - "Name": "__name__", - "DeclaringModuleId": null, - "QualifiedName": "__name__", - "IndexSpan": null - }, - { - "Value": null, - "Id": -1879833807, - "Name": "__qualname__", - "DeclaringModuleId": null, - "QualifiedName": "__qualname__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 174109373, - "Name": "__text_signature__", - "DeclaringModuleId": null, - "QualifiedName": "__text_signature__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 1521523639, - "Name": "__weakrefoffset__", - "DeclaringModuleId": null, - "QualifiedName": "__weakrefoffset__", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1144000522, - "Name": "_CDataMeta", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CDataMeta", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "source", - "Type": "t:bytearray", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "offset", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 3, - "Classes": [], - "Functions": [], - "Id": -1600044546, - "Name": "from_buffer", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CData.from_buffer", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "source", - "Type": "t:bytearray", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "offset", - "Type": "t:int", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 3, - "Classes": [], - "Functions": [], - "Id": 1037546774, - "Name": "from_buffer_copy", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CData.from_buffer_copy", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "address", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 3, - "Classes": [], - "Functions": [], - "Id": 562537974, - "Name": "from_address", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CData.from_address", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "obj", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:typing:Union[Unknown, ctypes(stub):_CArgObject]$typing(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 3, - "Classes": [], - "Functions": [], - "Id": -177816817, - "Name": "from_param", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CData.from_param", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "cls", - "Type": "t:typing:Type[Unknown]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "library", - "Type": "t:ctypes:CDLL$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": null, - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 3, - "Classes": [], - "Functions": [], - "Id": 2048326049, - "Name": "in_dll", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CData.in_dll", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1721516133, - "Name": "_CData", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CData", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_CData$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -638871764, - "Name": "_PointerLike", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_PointerLike", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_PointerLike$ctypes(3.7)", - "t:ctypes(stub):_CData$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "address", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "callable", - "Type": null, - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "func_spec", - "Type": "t:typing:Tuple[typing:Union[str, int], ctypes(stub):CDLL]$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "paramflags", - "Type": "t:typing:Tuple[Unknown]$typing(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - }, - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "vtlb_index", - "Type": "t:int", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "paramflags", - "Type": "t:typing:Tuple[Unknown]$typing(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - }, - { - "Name": "iid", - "Type": "t:ctypes(stub):pointer$ctypes(3.7)", - "DefaultValue": "i:...", - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_FuncPointer.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_FuncPointer$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "args", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 1 - }, - { - "Name": "kwargs", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 2 - } - ], - "ReturnType": "i:typing:Any$typing(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 782173109, - "Name": "__call__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_FuncPointer.__call__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:typing:Union[typing:Type[ctypes(stub):_CData], Unknown, None]$typing(3.7)", - "Id": -1767505949, - "Name": "restype", - "DeclaringModuleId": null, - "QualifiedName": "restype", - "IndexSpan": null - }, - { - "Value": "i:typing:Sequence[typing:Type[ctypes(stub):_CData]]$typing(3.7)", - "Id": -212017926, - "Name": "argtypes", - "DeclaringModuleId": null, - "QualifiedName": "argtypes", - "IndexSpan": null - }, - { - "Value": null, - "Id": -1532007462, - "Name": "errcheck", - "DeclaringModuleId": null, - "QualifiedName": "errcheck", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -778081647, - "Name": "_FuncPointer", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_FuncPointer", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1648730790, - "Name": "_CArgObject", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CArgObject", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:object" - ], - "NamedTupleBases": [], - "Methods": [], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -2082031766, - "Name": "offset", - "DeclaringModuleId": null, - "QualifiedName": "offset", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": 24771736, - "Name": "size", - "DeclaringModuleId": null, - "QualifiedName": "size", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": 1829463967, - "Name": "_CField", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_CField", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": "type(object_or_name, bases, dict)\ntype(object) -> the object's type\ntype(name, bases, dict) -> a new type", - "Bases": [ - "t:ctypes(stub):_CDataMeta$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_StructUnionMeta$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:ctypes(stub):_CField$ctypes(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -521760752, - "Name": "__getattr__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_StructUnionMeta.__getattr__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:typing:Sequence[typing:Union[typing:Tuple[str, typing:Type[ctypes(stub):_CData]], typing:Tuple[str, typing:Type[ctypes(stub):_CData], int]]]$typing(3.7)", - "Id": -1419773794, - "Name": "_fields_", - "DeclaringModuleId": null, - "QualifiedName": "_fields_", - "IndexSpan": null - }, - { - "Value": "i:int", - "Id": 1763940158, - "Name": "_pack_", - "DeclaringModuleId": null, - "QualifiedName": "_pack_", - "IndexSpan": null - }, - { - "Value": "i:typing:Sequence[str]$typing(3.7)", - "Id": -783398214, - "Name": "_anonymous_", - "DeclaringModuleId": null, - "QualifiedName": "_anonymous_", - "IndexSpan": null - }, - { - "Value": null, - "Id": -1388753224, - "Name": "__basicsize__", - "DeclaringModuleId": null, - "QualifiedName": "__basicsize__", - "IndexSpan": null - }, - { - "Value": null, - "Id": -198224608, - "Name": "__dictoffset__", - "DeclaringModuleId": null, - "QualifiedName": "__dictoffset__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 1444705936, - "Name": "__flags__", - "DeclaringModuleId": null, - "QualifiedName": "__flags__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 919460331, - "Name": "__itemsize__", - "DeclaringModuleId": null, - "QualifiedName": "__itemsize__", - "IndexSpan": null - }, - { - "Value": "i:list$ctypes(3.7)", - "Id": -1627592461, - "Name": "__mro__", - "DeclaringModuleId": null, - "QualifiedName": "__mro__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 1097116834, - "Name": "__name__", - "DeclaringModuleId": null, - "QualifiedName": "__name__", - "IndexSpan": null - }, - { - "Value": null, - "Id": -1879833807, - "Name": "__qualname__", - "DeclaringModuleId": null, - "QualifiedName": "__qualname__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 174109373, - "Name": "__text_signature__", - "DeclaringModuleId": null, - "QualifiedName": "__text_signature__", - "IndexSpan": null - }, - { - "Value": null, - "Id": 1521523639, - "Name": "__weakrefoffset__", - "DeclaringModuleId": null, - "QualifiedName": "__weakrefoffset__", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1026689833, - "Name": "_StructUnionMeta", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_StructUnionMeta", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Documentation": null, - "Bases": [ - "t:ctypes(stub):_CData$ctypes(3.7)", - "t:object" - ], - "NamedTupleBases": [], - "Methods": [ - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "args", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 1 - }, - { - "Name": "kw", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 2 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": 965872103, - "Name": "__init__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_StructUnionBase.__init__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:typing:Any$typing(3.7)", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -521760752, - "Name": "__getattr__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_StructUnionBase.__getattr__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - }, - { - "Overloads": [ - { - "Parameters": [ - { - "Name": "self", - "Type": "t:ctypes(stub):_StructUnionBase$ctypes(3.7)", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "name", - "Type": "t:str", - "DefaultValue": null, - "Kind": 0 - }, - { - "Name": "value", - "Type": "t:typing:Any$typing(3.7)", - "DefaultValue": null, - "Kind": 0 - } - ], - "ReturnType": "i:None", - "Documentation": null - } - ], - "Documentation": null, - "Attributes": 0, - "Classes": [], - "Functions": [], - "Id": -736377828, - "Name": "__setattr__", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_StructUnionBase.__setattr__", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "Properties": [], - "Fields": [ - { - "Value": "i:int", - "Id": -1555123786, - "Name": "_b_base", - "DeclaringModuleId": null, - "QualifiedName": "_b_base", - "IndexSpan": null - }, - { - "Value": "i:bool", - "Id": -422546917, - "Name": "_b_needsfree_", - "DeclaringModuleId": null, - "QualifiedName": "_b_needsfree_", - "IndexSpan": null - }, - { - "Value": "i:typing:Mapping[typing:Any, int]$typing(3.7)", - "Id": 2076796140, - "Name": "_objects", - "DeclaringModuleId": null, - "QualifiedName": "_objects", - "IndexSpan": null - } - ], - "Classes": [], - "GenericBaseParameters": [], - "GenericParameterValues": [], - "Id": -1027021405, - "Name": "_StructUnionBase", - "DeclaringModuleId": "ctypes(3.7)", - "QualifiedName": "ctypes(stub):_StructUnionBase", - "IndexSpan": { - "Start": 0, - "Length": 0 - } - } - ], - "TypeVars": [ - { - "Constraints": [], - "Bound": null, - "Covariant": null, - "Contravariant": null, - "Id": 25132, - "Name": "_T", - "DeclaringModuleId": null, - "QualifiedName": "ctypes:_T", - "IndexSpan": null - }, - { - "Constraints": [], - "Bound": "ctypes:CDLL", - "Covariant": null, - "Contravariant": null, - "Id": 748306232, - "Name": "_DLLT", - "DeclaringModuleId": null, - "QualifiedName": "ctypes:_DLLT", - "IndexSpan": null - }, - { - "Constraints": [], - "Bound": "ctypes(stub):_CData", - "Covariant": null, - "Contravariant": null, - "Id": 778649, - "Name": "_CT", - "DeclaringModuleId": null, - "QualifiedName": "ctypes:_CT", - "IndexSpan": null - }, - { - "Constraints": [], - "Bound": "ctypes(stub):_PointerLike", - "Covariant": null, - "Contravariant": null, - "Id": 779052, - "Name": "_PT", - "DeclaringModuleId": null, - "QualifiedName": "ctypes:_PT", - "IndexSpan": null - } - ], - "NamedTuples": [], - "NewLines": [ - { - "EndIndex": 52, - "Kind": 3 - }, - { - "EndIndex": 54, - "Kind": 3 - }, - { - "EndIndex": 85, - "Kind": 3 - }, - { - "EndIndex": 87, - "Kind": 3 - }, - { - "EndIndex": 110, - "Kind": 3 - }, - { - "EndIndex": 112, - "Kind": 3 - }, - { - "EndIndex": 157, - "Kind": 3 - }, - { - "EndIndex": 187, - "Kind": 3 - }, - { - "EndIndex": 230, - "Kind": 3 - }, - { - "EndIndex": 282, - "Kind": 3 - }, - { - "EndIndex": 327, - "Kind": 3 - }, - { - "EndIndex": 362, - "Kind": 3 - }, - { - "EndIndex": 364, - "Kind": 3 - }, - { - "EndIndex": 406, - "Kind": 3 - }, - { - "EndIndex": 408, - "Kind": 3 - }, - { - "EndIndex": 444, - "Kind": 3 - }, - { - "EndIndex": 522, - "Kind": 3 - }, - { - "EndIndex": 524, - "Kind": 3 - }, - { - "EndIndex": 546, - "Kind": 3 - }, - { - "EndIndex": 583, - "Kind": 3 - }, - { - "EndIndex": 585, - "Kind": 3 - }, - { - "EndIndex": 612, - "Kind": 3 - }, - { - "EndIndex": 667, - "Kind": 3 - }, - { - "EndIndex": 723, - "Kind": 3 - }, - { - "EndIndex": 780, - "Kind": 3 - }, - { - "EndIndex": 838, - "Kind": 3 - }, - { - "EndIndex": 851, - "Kind": 3 - }, - { - "EndIndex": 853, - "Kind": 3 - }, - { - "EndIndex": 905, - "Kind": 3 - }, - { - "EndIndex": 941, - "Kind": 3 - }, - { - "EndIndex": 943, - "Kind": 3 - }, - { - "EndIndex": 1001, - "Kind": 3 - }, - { - "EndIndex": 1052, - "Kind": 3 - }, - { - "EndIndex": 1103, - "Kind": 3 - }, - { - "EndIndex": 1159, - "Kind": 3 - }, - { - "EndIndex": 1161, - "Kind": 3 - }, - { - "EndIndex": 1185, - "Kind": 3 - }, - { - "EndIndex": 1205, - "Kind": 3 - }, - { - "EndIndex": 1208, - "Kind": 3 - }, - { - "EndIndex": 1229, - "Kind": 3 - }, - { - "EndIndex": 1232, - "Kind": 3 - }, - { - "EndIndex": 1251, - "Kind": 3 - }, - { - "EndIndex": 1277, - "Kind": 3 - }, - { - "EndIndex": 1280, - "Kind": 3 - }, - { - "EndIndex": 1298, - "Kind": 3 - }, - { - "EndIndex": 1300, - "Kind": 3 - }, - { - "EndIndex": 1344, - "Kind": 3 - }, - { - "EndIndex": 1400, - "Kind": 3 - }, - { - "EndIndex": 1456, - "Kind": 3 - }, - { - "EndIndex": 1520, - "Kind": 3 - }, - { - "EndIndex": 1529, - "Kind": 3 - }, - { - "EndIndex": 1562, - "Kind": 3 - }, - { - "EndIndex": 1588, - "Kind": 3 - }, - { - "EndIndex": 1620, - "Kind": 3 - }, - { - "EndIndex": 1653, - "Kind": 3 - }, - { - "EndIndex": 1678, - "Kind": 3 - }, - { - "EndIndex": 1704, - "Kind": 3 - }, - { - "EndIndex": 1724, - "Kind": 3 - }, - { - "EndIndex": 1757, - "Kind": 3 - }, - { - "EndIndex": 1790, - "Kind": 3 - }, - { - "EndIndex": 1815, - "Kind": 3 - }, - { - "EndIndex": 1835, - "Kind": 3 - }, - { - "EndIndex": 1862, - "Kind": 3 - }, - { - "EndIndex": 1864, - "Kind": 3 - }, - { - "EndIndex": 1896, - "Kind": 3 - }, - { - "EndIndex": 1950, - "Kind": 3 - }, - { - "EndIndex": 1973, - "Kind": 3 - }, - { - "EndIndex": 2054, - "Kind": 3 - }, - { - "EndIndex": 2109, - "Kind": 3 - }, - { - "EndIndex": 2154, - "Kind": 3 - }, - { - "EndIndex": 2156, - "Kind": 3 - }, - { - "EndIndex": 2180, - "Kind": 3 - }, - { - "EndIndex": 2222, - "Kind": 3 - }, - { - "EndIndex": 2260, - "Kind": 3 - }, - { - "EndIndex": 2340, - "Kind": 3 - }, - { - "EndIndex": 2342, - "Kind": 3 - }, - { - "EndIndex": 2372, - "Kind": 3 - }, - { - "EndIndex": 2428, - "Kind": 3 - }, - { - "EndIndex": 2430, - "Kind": 3 - }, - { - "EndIndex": 2502, - "Kind": 3 - }, - { - "EndIndex": 2524, - "Kind": 3 - }, - { - "EndIndex": 2526, - "Kind": 3 - }, - { - "EndIndex": 2578, - "Kind": 3 - }, - { - "EndIndex": 2660, - "Kind": 3 - }, - { - "EndIndex": 2758, - "Kind": 3 - }, - { - "EndIndex": 2857, - "Kind": 3 - }, - { - "EndIndex": 2952, - "Kind": 3 - }, - { - "EndIndex": 2961, - "Kind": 3 - }, - { - "EndIndex": 2990, - "Kind": 3 - }, - { - "EndIndex": 3026, - "Kind": 3 - }, - { - "EndIndex": 3064, - "Kind": 3 - }, - { - "EndIndex": 3105, - "Kind": 3 - }, - { - "EndIndex": 3147, - "Kind": 3 - }, - { - "EndIndex": 3159, - "Kind": 3 - }, - { - "EndIndex": 3234, - "Kind": 3 - }, - { - "EndIndex": 3244, - "Kind": 3 - }, - { - "EndIndex": 3306, - "Kind": 3 - }, - { - "EndIndex": 3328, - "Kind": 3 - }, - { - "EndIndex": 3369, - "Kind": 3 - }, - { - "EndIndex": 3404, - "Kind": 3 - }, - { - "EndIndex": 3437, - "Kind": 3 - }, - { - "EndIndex": 3466, - "Kind": 3 - }, - { - "EndIndex": 3537, - "Kind": 3 - }, - { - "EndIndex": 3567, - "Kind": 3 - }, - { - "EndIndex": 3569, - "Kind": 3 - }, - { - "EndIndex": 3591, - "Kind": 3 - }, - { - "EndIndex": 3639, - "Kind": 3 - }, - { - "EndIndex": 3702, - "Kind": 3 - }, - { - "EndIndex": 3704, - "Kind": 3 - }, - { - "EndIndex": 3734, - "Kind": 3 - }, - { - "EndIndex": 3782, - "Kind": 3 - }, - { - "EndIndex": 3849, - "Kind": 3 - }, - { - "EndIndex": 3884, - "Kind": 3 - }, - { - "EndIndex": 3924, - "Kind": 3 - }, - { - "EndIndex": 3966, - "Kind": 3 - }, - { - "EndIndex": 4011, - "Kind": 3 - }, - { - "EndIndex": 4057, - "Kind": 3 - }, - { - "EndIndex": 4073, - "Kind": 3 - }, - { - "EndIndex": 4152, - "Kind": 3 - }, - { - "EndIndex": 4166, - "Kind": 3 - }, - { - "EndIndex": 4234, - "Kind": 3 - }, - { - "EndIndex": 4260, - "Kind": 3 - }, - { - "EndIndex": 4307, - "Kind": 3 - }, - { - "EndIndex": 4346, - "Kind": 3 - }, - { - "EndIndex": 4383, - "Kind": 3 - }, - { - "EndIndex": 4416, - "Kind": 3 - }, - { - "EndIndex": 4495, - "Kind": 3 - }, - { - "EndIndex": 4531, - "Kind": 3 - }, - { - "EndIndex": 4560, - "Kind": 3 - }, - { - "EndIndex": 4645, - "Kind": 3 - }, - { - "EndIndex": 4647, - "Kind": 3 - }, - { - "EndIndex": 4674, - "Kind": 3 - }, - { - "EndIndex": 4717, - "Kind": 3 - }, - { - "EndIndex": 4719, - "Kind": 3 - }, - { - "EndIndex": 4784, - "Kind": 3 - }, - { - "EndIndex": 4826, - "Kind": 3 - }, - { - "EndIndex": 4860, - "Kind": 3 - }, - { - "EndIndex": 4862, - "Kind": 3 - }, - { - "EndIndex": 4900, - "Kind": 3 - }, - { - "EndIndex": 4967, - "Kind": 3 - }, - { - "EndIndex": 5030, - "Kind": 3 - }, - { - "EndIndex": 5063, - "Kind": 3 - }, - { - "EndIndex": 5089, - "Kind": 3 - }, - { - "EndIndex": 5149, - "Kind": 3 - }, - { - "EndIndex": 5180, - "Kind": 3 - }, - { - "EndIndex": 5236, - "Kind": 3 - }, - { - "EndIndex": 5264, - "Kind": 3 - }, - { - "EndIndex": 5332, - "Kind": 3 - }, - { - "EndIndex": 5384, - "Kind": 3 - }, - { - "EndIndex": 5386, - "Kind": 3 - }, - { - "EndIndex": 5418, - "Kind": 3 - }, - { - "EndIndex": 5436, - "Kind": 3 - }, - { - "EndIndex": 5461, - "Kind": 3 - }, - { - "EndIndex": 5475, - "Kind": 3 - }, - { - "EndIndex": 5514, - "Kind": 3 - }, - { - "EndIndex": 5542, - "Kind": 3 - }, - { - "EndIndex": 5597, - "Kind": 3 - }, - { - "EndIndex": 5626, - "Kind": 3 - }, - { - "EndIndex": 5628, - "Kind": 3 - }, - { - "EndIndex": 5658, - "Kind": 3 - }, - { - "EndIndex": 5676, - "Kind": 3 - }, - { - "EndIndex": 5698, - "Kind": 3 - }, - { - "EndIndex": 5700, - "Kind": 3 - }, - { - "EndIndex": 5731, - "Kind": 3 - }, - { - "EndIndex": 5749, - "Kind": 3 - }, - { - "EndIndex": 5772, - "Kind": 3 - }, - { - "EndIndex": 5774, - "Kind": 3 - }, - { - "EndIndex": 5803, - "Kind": 3 - }, - { - "EndIndex": 5821, - "Kind": 3 - }, - { - "EndIndex": 5842, - "Kind": 3 - }, - { - "EndIndex": 5844, - "Kind": 3 - }, - { - "EndIndex": 5874, - "Kind": 3 - }, - { - "EndIndex": 5892, - "Kind": 3 - }, - { - "EndIndex": 5914, - "Kind": 3 - }, - { - "EndIndex": 5916, - "Kind": 3 - }, - { - "EndIndex": 5954, - "Kind": 3 - }, - { - "EndIndex": 6028, - "Kind": 3 - }, - { - "EndIndex": 6048, - "Kind": 3 - }, - { - "EndIndex": 6070, - "Kind": 3 - }, - { - "EndIndex": 6077, - "Kind": 3 - }, - { - "EndIndex": 6109, - "Kind": 3 - }, - { - "EndIndex": 6131, - "Kind": 3 - }, - { - "EndIndex": 6155, - "Kind": 3 - }, - { - "EndIndex": 6157, - "Kind": 3 - }, - { - "EndIndex": 6190, - "Kind": 3 - }, - { - "EndIndex": 6212, - "Kind": 3 - }, - { - "EndIndex": 6237, - "Kind": 3 - }, - { - "EndIndex": 6239, - "Kind": 3 - }, - { - "EndIndex": 6269, - "Kind": 3 - }, - { - "EndIndex": 6287, - "Kind": 3 - }, - { - "EndIndex": 6309, - "Kind": 3 - }, - { - "EndIndex": 6311, - "Kind": 3 - }, - { - "EndIndex": 6342, - "Kind": 3 - }, - { - "EndIndex": 6360, - "Kind": 3 - }, - { - "EndIndex": 6383, - "Kind": 3 - }, - { - "EndIndex": 6385, - "Kind": 3 - }, - { - "EndIndex": 6420, - "Kind": 3 - }, - { - "EndIndex": 6438, - "Kind": 3 - }, - { - "EndIndex": 6484, - "Kind": 3 - }, - { - "EndIndex": 6513, - "Kind": 3 - }, - { - "EndIndex": 6515, - "Kind": 3 - }, - { - "EndIndex": 6553, - "Kind": 3 - }, - { - "EndIndex": 6638, - "Kind": 3 - }, - { - "EndIndex": 6663, - "Kind": 3 - }, - { - "EndIndex": 6690, - "Kind": 3 - }, - { - "EndIndex": 6697, - "Kind": 3 - }, - { - "EndIndex": 6734, - "Kind": 3 - }, - { - "EndIndex": 6756, - "Kind": 3 - }, - { - "EndIndex": 6785, - "Kind": 3 - }, - { - "EndIndex": 6787, - "Kind": 3 - }, - { - "EndIndex": 6825, - "Kind": 3 - }, - { - "EndIndex": 6847, - "Kind": 3 - }, - { - "EndIndex": 6884, - "Kind": 3 - }, - { - "EndIndex": 6929, - "Kind": 3 - }, - { - "EndIndex": 6977, - "Kind": 3 - }, - { - "EndIndex": 7007, - "Kind": 3 - }, - { - "EndIndex": 7009, - "Kind": 3 - }, - { - "EndIndex": 7039, - "Kind": 3 - }, - { - "EndIndex": 7057, - "Kind": 3 - }, - { - "EndIndex": 7112, - "Kind": 3 - }, - { - "EndIndex": 7139, - "Kind": 3 - }, - { - "EndIndex": 7160, - "Kind": 3 - }, - { - "EndIndex": 7182, - "Kind": 3 - }, - { - "EndIndex": 7184, - "Kind": 3 - }, - { - "EndIndex": 7213, - "Kind": 3 - }, - { - "EndIndex": 7231, - "Kind": 3 - }, - { - "EndIndex": 7283, - "Kind": 3 - }, - { - "EndIndex": 7304, - "Kind": 3 - }, - { - "EndIndex": 7306, - "Kind": 3 - }, - { - "EndIndex": 7335, - "Kind": 3 - }, - { - "EndIndex": 7353, - "Kind": 3 - }, - { - "EndIndex": 7405, - "Kind": 3 - }, - { - "EndIndex": 7426, - "Kind": 3 - }, - { - "EndIndex": 7428, - "Kind": 3 - }, - { - "EndIndex": 7459, - "Kind": 3 - }, - { - "EndIndex": 7477, - "Kind": 3 - }, - { - "EndIndex": 7502, - "Kind": 3 - }, - { - "EndIndex": 7589, - "Kind": 3 - }, - { - "EndIndex": 7617, - "Kind": 3 - }, - { - "EndIndex": 7619, - "Kind": 3 - }, - { - "EndIndex": 7650, - "Kind": 3 - }, - { - "EndIndex": 7668, - "Kind": 3 - }, - { - "EndIndex": 7725, - "Kind": 3 - }, - { - "EndIndex": 7748, - "Kind": 3 - }, - { - "EndIndex": 7750, - "Kind": 3 - }, - { - "EndIndex": 7779, - "Kind": 3 - }, - { - "EndIndex": 7797, - "Kind": 3 - }, - { - "EndIndex": 7799, - "Kind": 3 - }, - { - "EndIndex": 7858, - "Kind": 3 - }, - { - "EndIndex": 7860, - "Kind": 3 - }, - { - "EndIndex": 7892, - "Kind": 3 - }, - { - "EndIndex": 7910, - "Kind": 3 - }, - { - "EndIndex": 7935, - "Kind": 3 - }, - { - "EndIndex": 8022, - "Kind": 3 - }, - { - "EndIndex": 8024, - "Kind": 3 - }, - { - "EndIndex": 8054, - "Kind": 3 - }, - { - "EndIndex": 8072, - "Kind": 3 - }, - { - "EndIndex": 8074, - "Kind": 3 - }, - { - "EndIndex": 8095, - "Kind": 3 - }, - { - "EndIndex": 8128, - "Kind": 3 - }, - { - "EndIndex": 8159, - "Kind": 3 - }, - { - "EndIndex": 8185, - "Kind": 3 - }, - { - "EndIndex": 8222, - "Kind": 3 - }, - { - "EndIndex": 8263, - "Kind": 3 - }, - { - "EndIndex": 8319, - "Kind": 3 - }, - { - "EndIndex": 8359, - "Kind": 3 - }, - { - "EndIndex": 8413, - "Kind": 3 - }, - { - "EndIndex": 8455, - "Kind": 3 - }, - { - "EndIndex": 8530, - "Kind": 3 - }, - { - "EndIndex": 8605, - "Kind": 3 - }, - { - "EndIndex": 8677, - "Kind": 3 - }, - { - "EndIndex": 8750, - "Kind": 3 - }, - { - "EndIndex": 8786, - "Kind": 3 - }, - { - "EndIndex": 8788, - "Kind": 3 - }, - { - "EndIndex": 8833, - "Kind": 3 - }, - { - "EndIndex": 8891, - "Kind": 3 - }, - { - "EndIndex": 8948, - "Kind": 3 - }, - { - "EndIndex": 9014, - "Kind": 3 - }, - { - "EndIndex": 9023, - "Kind": 3 - }, - { - "EndIndex": 9054, - "Kind": 3 - }, - { - "EndIndex": 9080, - "Kind": 3 - }, - { - "EndIndex": 9112, - "Kind": 3 - }, - { - "EndIndex": 9146, - "Kind": 3 - }, - { - "EndIndex": 9171, - "Kind": 3 - }, - { - "EndIndex": 9197, - "Kind": 3 - }, - { - "EndIndex": 9217, - "Kind": 3 - }, - { - "EndIndex": 9250, - "Kind": 3 - }, - { - "EndIndex": 9284, - "Kind": 3 - }, - { - "EndIndex": 9309, - "Kind": 3 - }, - { - "EndIndex": 9329, - "Kind": 3 - }, - { - "EndIndex": 9356, - "Kind": 3 - }, - { - "EndIndex": 9358, - "Kind": 3 - }, - { - "EndIndex": 9360, - "Kind": 3 - }, - { - "EndIndex": 9378, - "Kind": 3 - }, - { - "EndIndex": 9413, - "Kind": 3 - }, - { - "EndIndex": 9469, - "Kind": 3 - }, - { - "EndIndex": 9538, - "Kind": 3 - }, - { - "EndIndex": 9586, - "Kind": 3 - }, - { - "EndIndex": 9632, - "Kind": 3 - }, - { - "EndIndex": 9659, - "Kind": 3 - }, - { - "EndIndex": 9699, - "Kind": 3 - }, - { - "EndIndex": 9741, - "Kind": 3 - }, - { - "EndIndex": 9743, - "Kind": 3 - }, - { - "EndIndex": 9761, - "Kind": 3 - }, - { - "EndIndex": 9783, - "Kind": 3 - }, - { - "EndIndex": 9805, - "Kind": 3 - }, - { - "EndIndex": 9807, - "Kind": 3 - }, - { - "EndIndex": 9873, - "Kind": 3 - }, - { - "EndIndex": 9875, - "Kind": 3 - }, - { - "EndIndex": 9877, - "Kind": 3 - }, - { - "EndIndex": 9898, - "Kind": 3 - }, - { - "EndIndex": 9963, - "Kind": 3 - }, - { - "EndIndex": 10026, - "Kind": 3 - }, - { - "EndIndex": 10070, - "Kind": 3 - }, - { - "EndIndex": 10072, - "Kind": 3 - }, - { - "EndIndex": 10137, - "Kind": 3 - }, - { - "EndIndex": 10186, - "Kind": 3 - }, - { - "EndIndex": 10188, - "Kind": 3 - }, - { - "EndIndex": 10224, - "Kind": 3 - }, - { - "EndIndex": 10263, - "Kind": 3 - }, - { - "EndIndex": 10265, - "Kind": 3 - }, - { - "EndIndex": 10336, - "Kind": 3 - }, - { - "EndIndex": 10367, - "Kind": 3 - }, - { - "EndIndex": 10376, - "Kind": 3 - }, - { - "EndIndex": 10412, - "Kind": 3 - }, - { - "EndIndex": 10440, - "Kind": 3 - }, - { - "EndIndex": 10471, - "Kind": 3 - }, - { - "EndIndex": 10502, - "Kind": 3 - }, - { - "EndIndex": 10519, - "Kind": 3 - }, - { - "EndIndex": 10540, - "Kind": 3 - }, - { - "EndIndex": 10542, - "Kind": 3 - }, - { - "EndIndex": 10604, - "Kind": 3 - }, - { - "EndIndex": 10639, - "Kind": 3 - }, - { - "EndIndex": 10680, - "Kind": 3 - }, - { - "EndIndex": 10707, - "Kind": 3 - }, - { - "EndIndex": 10742, - "Kind": 3 - }, - { - "EndIndex": 10765, - "Kind": 3 - }, - { - "EndIndex": 10807, - "Kind": 3 - }, - { - "EndIndex": 10835, - "Kind": 3 - }, - { - "EndIndex": 10881, - "Kind": 3 - }, - { - "EndIndex": 10926, - "Kind": 3 - }, - { - "EndIndex": 10990, - "Kind": 3 - }, - { - "EndIndex": 11059, - "Kind": 3 - }, - { - "EndIndex": 11138, - "Kind": 3 - }, - { - "EndIndex": 11215, - "Kind": 3 - }, - { - "EndIndex": 11232, - "Kind": 3 - }, - { - "EndIndex": 11295, - "Kind": 3 - }, - { - "EndIndex": 11355, - "Kind": 3 - }, - { - "EndIndex": 11357, - "Kind": 3 - }, - { - "EndIndex": 11393, - "Kind": 3 - }, - { - "EndIndex": 11422, - "Kind": 3 - }, - { - "EndIndex": 11467, - "Kind": 3 - }, - { - "EndIndex": 11501, - "Kind": 3 - }, - { - "EndIndex": 11503, - "Kind": 3 - }, - { - "EndIndex": 11531, - "Kind": 3 - }, - { - "EndIndex": 11585, - "Kind": 3 - }, - { - "EndIndex": 11600, - "Kind": 3 - }, - { - "EndIndex": 11635, - "Kind": 3 - }, - { - "EndIndex": 11637, - "Kind": 3 - }, - { - "EndIndex": 11662, - "Kind": 3 - }, - { - "EndIndex": 11712, - "Kind": 3 - }, - { - "EndIndex": 11766, - "Kind": 3 - }, - { - "EndIndex": 11822, - "Kind": 3 - }, - { - "EndIndex": 11872, - "Kind": 3 - }, - { - "EndIndex": 11874, - "Kind": 3 - }, - { - "EndIndex": 11908, - "Kind": 3 - }, - { - "EndIndex": 11967, - "Kind": 3 - }, - { - "EndIndex": 12007, - "Kind": 3 - }, - { - "EndIndex": 12046, - "Kind": 3 - }, - { - "EndIndex": 12081, - "Kind": 3 - }, - { - "EndIndex": 12102, - "Kind": 3 - }, - { - "EndIndex": 12104, - "Kind": 3 - }, - { - "EndIndex": 12149, - "Kind": 3 - }, - { - "EndIndex": 12204, - "Kind": 3 - }, - { - "EndIndex": 12254, - "Kind": 3 - }, - { - "EndIndex": 12299, - "Kind": 3 - }, - { - "EndIndex": 12320, - "Kind": 3 - }, - { - "EndIndex": 12322, - "Kind": 3 - }, - { - "EndIndex": 12342, - "Kind": 3 - }, - { - "EndIndex": 12410, - "Kind": 3 - }, - { - "EndIndex": 12477, - "Kind": 3 - }, - { - "EndIndex": 12523, - "Kind": 3 - }, - { - "EndIndex": 12532, - "Kind": 3 - }, - { - "EndIndex": 12590, - "Kind": 3 - }, - { - "EndIndex": 12592, - "Kind": 3 - }, - { - "EndIndex": 12614, - "Kind": 3 - }, - { - "EndIndex": 12616, - "Kind": 3 - }, - { - "EndIndex": 12641, - "Kind": 3 - }, - { - "EndIndex": 12711, - "Kind": 3 - }, - { - "EndIndex": 12756, - "Kind": 3 - }, - { - "EndIndex": 12769, - "Kind": 3 - }, - { - "EndIndex": 12811, - "Kind": 3 - }, - { - "EndIndex": 12813, - "Kind": 3 - }, - { - "EndIndex": 12868, - "Kind": 3 - }, - { - "EndIndex": 12911, - "Kind": 3 - }, - { - "EndIndex": 12965, - "Kind": 3 - }, - { - "EndIndex": 12999, - "Kind": 3 - }, - { - "EndIndex": 13021, - "Kind": 3 - }, - { - "EndIndex": 13092, - "Kind": 3 - }, - { - "EndIndex": 13158, - "Kind": 3 - }, - { - "EndIndex": 13201, - "Kind": 3 - }, - { - "EndIndex": 13212, - "Kind": 3 - }, - { - "EndIndex": 13282, - "Kind": 3 - }, - { - "EndIndex": 13351, - "Kind": 3 - }, - { - "EndIndex": 13421, - "Kind": 3 - }, - { - "EndIndex": 13489, - "Kind": 3 - }, - { - "EndIndex": 13508, - "Kind": 3 - }, - { - "EndIndex": 13549, - "Kind": 3 - }, - { - "EndIndex": 13551, - "Kind": 3 - }, - { - "EndIndex": 13576, - "Kind": 3 - }, - { - "EndIndex": 13646, - "Kind": 3 - }, - { - "EndIndex": 13714, - "Kind": 3 - }, - { - "EndIndex": 13780, - "Kind": 3 - }, - { - "EndIndex": 13801, - "Kind": 3 - }, - { - "EndIndex": 13814, - "Kind": 3 - }, - { - "EndIndex": 13856, - "Kind": 3 - }, - { - "EndIndex": 13890, - "Kind": 3 - }, - { - "EndIndex": 13892, - "Kind": 3 - }, - { - "EndIndex": 13922, - "Kind": 3 - }, - { - "EndIndex": 13956, - "Kind": 3 - }, - { - "EndIndex": 13989, - "Kind": 3 - }, - { - "EndIndex": 13991, - "Kind": 3 - }, - { - "EndIndex": 14025, - "Kind": 3 - }, - { - "EndIndex": 14053, - "Kind": 3 - }, - { - "EndIndex": 14093, - "Kind": 3 - }, - { - "EndIndex": 14128, - "Kind": 3 - }, - { - "EndIndex": 14162, - "Kind": 3 - }, - { - "EndIndex": 14182, - "Kind": 3 - }, - { - "EndIndex": 14184, - "Kind": 3 - }, - { - "EndIndex": 14218, - "Kind": 3 - }, - { - "EndIndex": 14254, - "Kind": 3 - }, - { - "EndIndex": 14256, - "Kind": 3 - }, - { - "EndIndex": 14290, - "Kind": 3 - }, - { - "EndIndex": 14326, - "Kind": 3 - }, - { - "EndIndex": 14328, - "Kind": 3 - }, - { - "EndIndex": 14356, - "Kind": 3 - }, - { - "EndIndex": 14386, - "Kind": 3 - }, - { - "EndIndex": 14388, - "Kind": 3 - }, - { - "EndIndex": 14410, - "Kind": 3 - }, - { - "EndIndex": 14469, - "Kind": 3 - }, - { - "EndIndex": 14502, - "Kind": 3 - }, - { - "EndIndex": 14571, - "Kind": 3 - }, - { - "EndIndex": 14578, - "Kind": 3 - }, - { - "EndIndex": 14607, - "Kind": 3 - }, - { - "EndIndex": 14609, - "Kind": 3 - }, - { - "EndIndex": 14611, - "Kind": 3 - }, - { - "EndIndex": 14633, - "Kind": 3 - }, - { - "EndIndex": 14669, - "Kind": 3 - }, - { - "EndIndex": 14705, - "Kind": 3 - }, - { - "EndIndex": 14707, - "Kind": 3 - }, - { - "EndIndex": 14756, - "Kind": 3 - }, - { - "EndIndex": 14812, - "Kind": 3 - }, - { - "EndIndex": 14814, - "Kind": 3 - }, - { - "EndIndex": 14856, - "Kind": 3 - }, - { - "EndIndex": 14882, - "Kind": 3 - }, - { - "EndIndex": 14917, - "Kind": 3 - }, - { - "EndIndex": 14944, - "Kind": 3 - }, - { - "EndIndex": 14991, - "Kind": 3 - }, - { - "EndIndex": 15040, - "Kind": 3 - }, - { - "EndIndex": 15042, - "Kind": 3 - }, - { - "EndIndex": 15082, - "Kind": 3 - }, - { - "EndIndex": 15105, - "Kind": 3 - }, - { - "EndIndex": 15128, - "Kind": 3 - }, - { - "EndIndex": 15171, - "Kind": 3 - }, - { - "EndIndex": 15195, - "Kind": 3 - }, - { - "EndIndex": 15219, - "Kind": 3 - }, - { - "EndIndex": 15266, - "Kind": 3 - }, - { - "EndIndex": 15294, - "Kind": 3 - }, - { - "EndIndex": 15322, - "Kind": 3 - }, - { - "EndIndex": 15324, - "Kind": 3 - }, - { - "EndIndex": 15337, - "Kind": 3 - }, - { - "EndIndex": 15339, - "Kind": 3 - }, - { - "EndIndex": 15417, - "Kind": 3 - }, - { - "EndIndex": 15419, - "Kind": 3 - }, - { - "EndIndex": 15468, - "Kind": 3 - }, - { - "EndIndex": 15544, - "Kind": 3 - }, - { - "EndIndex": 15546, - "Kind": 3 - }, - { - "EndIndex": 15584, - "Kind": 3 - }, - { - "EndIndex": 15655, - "Kind": 3 - }, - { - "EndIndex": 15657, - "Kind": 3 - }, - { - "EndIndex": 15694, - "Kind": 3 - }, - { - "EndIndex": 15731, - "Kind": 3 - }, - { - "EndIndex": 15762, - "Kind": 3 - }, - { - "EndIndex": 15791, - "Kind": 3 - }, - { - "EndIndex": 15848, - "Kind": 3 - }, - { - "EndIndex": 15874, - "Kind": 3 - }, - { - "EndIndex": 15876, - "Kind": 3 - }, - { - "EndIndex": 15951, - "Kind": 3 - }, - { - "EndIndex": 15972, - "Kind": 3 - }, - { - "EndIndex": 16005, - "Kind": 3 - }, - { - "EndIndex": 16007, - "Kind": 3 - }, - { - "EndIndex": 16077, - "Kind": 3 - }, - { - "EndIndex": 16107, - "Kind": 3 - }, - { - "EndIndex": 16149, - "Kind": 3 - }, - { - "EndIndex": 16151, - "Kind": 3 - }, - { - "EndIndex": 16186, - "Kind": 3 - }, - { - "EndIndex": 16220, - "Kind": 3 - }, - { - "EndIndex": 16222, - "Kind": 3 - }, - { - "EndIndex": 16228, - "Kind": 3 - }, - { - "EndIndex": 16270, - "Kind": 3 - }, - { - "EndIndex": 16291, - "Kind": 3 - }, - { - "EndIndex": 16301, - "Kind": 3 - }, - { - "EndIndex": 16308, - "Kind": 3 - }, - { - "EndIndex": 16384, - "Kind": 3 - }, - { - "EndIndex": 16419, - "Kind": 3 - }, - { - "EndIndex": 16466, - "Kind": 3 - }, - { - "EndIndex": 16468, - "Kind": 3 - }, - { - "EndIndex": 16507, - "Kind": 3 - }, - { - "EndIndex": 16546, - "Kind": 3 - }, - { - "EndIndex": 16548, - "Kind": 3 - }, - { - "EndIndex": 16550, - "Kind": 3 - }, - { - "EndIndex": 16584, - "Kind": 3 - }, - { - "EndIndex": 16631, - "Kind": 3 - }, - { - "EndIndex": 16645, - "Kind": 3 - }, - { - "EndIndex": 16736, - "Kind": 3 - }, - { - "EndIndex": 16765, - "Kind": 3 - }, - { - "EndIndex": 16825, - "Kind": 3 - }, - { - "EndIndex": 16840, - "Kind": 3 - }, - { - "EndIndex": 16902, - "Kind": 3 - }, - { - "EndIndex": 16904, - "Kind": 3 - }, - { - "EndIndex": 16932, - "Kind": 3 - }, - { - "EndIndex": 16946, - "Kind": 3 - }, - { - "EndIndex": 17037, - "Kind": 3 - }, - { - "EndIndex": 17066, - "Kind": 3 - }, - { - "EndIndex": 17095, - "Kind": 3 - }, - { - "EndIndex": 17134, - "Kind": 3 - }, - { - "EndIndex": 17136, - "Kind": 3 - }, - { - "EndIndex": 17206, - "Kind": 3 - }, - { - "EndIndex": 17208, - "Kind": 3 - }, - { - "EndIndex": 17244, - "Kind": 3 - }, - { - "EndIndex": 17261, - "Kind": 3 - }, - { - "EndIndex": 17280, - "Kind": 3 - }, - { - "EndIndex": 17331, - "Kind": 3 - }, - { - "EndIndex": 17373, - "Kind": 3 - }, - { - "EndIndex": 17417, - "Kind": 3 - }, - { - "EndIndex": 17461, - "Kind": 3 - }, - { - "EndIndex": 17516, - "Kind": 3 - }, - { - "EndIndex": 17559, - "Kind": 3 - }, - { - "EndIndex": 17604, - "Kind": 3 - }, - { - "EndIndex": 17649, - "Kind": 3 - }, - { - "EndIndex": 17660, - "Kind": 3 - }, - { - "EndIndex": 17662, - "Kind": 3 - }, - { - "EndIndex": 17678, - "Kind": 3 - } - ], - "FileSize": 17678, - "Id": 585668072, - "Name": "ctypes", - "DeclaringModuleId": null, - "QualifiedName": "ctypes", - "IndexSpan": null -} \ No newline at end of file From 1d0b1baaf91220179bd9d2df87f43eff2359f4d2 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 10:07:22 -0800 Subject: [PATCH 043/102] Bump up db version --- src/Caching/Impl/ModuleDatabase.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index d36ba4440..3b5293244 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -18,7 +18,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; using LiteDB; @@ -60,7 +59,7 @@ public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { } public string CacheFolderBaseName => "analysis.v"; - public int DatabaseFormatVersion => 2; + public int DatabaseFormatVersion => 3; public string CacheFolder { get; } /// From 36aba0b3d4dd615ed13a4368a9e47909be63f77a Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 10:20:48 -0800 Subject: [PATCH 044/102] Bump db version to 5 --- src/Caching/Impl/ModuleDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 3b5293244..d3bf8bf4c 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -59,7 +59,7 @@ public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { } public string CacheFolderBaseName => "analysis.v"; - public int DatabaseFormatVersion => 3; + public int DatabaseFormatVersion => 5; public string CacheFolder { get; } /// From e15547035d9fcf726b665d588c7882887396944f Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 10:49:27 -0800 Subject: [PATCH 045/102] Test fix --- src/LanguageServer/Impl/LanguageServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index 5b5c0caa6..bf5608cfe 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -308,7 +308,7 @@ public async Task ExtensionCommand(JToken token, CancellationToken cancellationT public async Task ClearAnalysisCache(CancellationToken cancellationToken) { using (_requestTimer.Time("python/clearAnalysisCache")) using (await _prioritizer.ConfigurationPriorityAsync(cancellationToken)) { - Debug.Assert(_initialized); + // Debug.Assert(_initialized); _server.ClearAnalysisCache(); } } From e0de03ea138133b5108d7b6da46571a742e3acb1 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 11:14:30 -0800 Subject: [PATCH 046/102] Handle empty bases --- src/Caching/Impl/Models/ClassModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Caching/Impl/Models/ClassModel.cs b/src/Caching/Impl/Models/ClassModel.cs index b7b993403..86e76b530 100644 --- a/src/Caching/Impl/Models/ClassModel.cs +++ b/src/Caching/Impl/Models/ClassModel.cs @@ -106,10 +106,10 @@ public ClassModel(IPythonClassType cls, IServiceContainer services) { IndexSpan = cls.Location.IndexSpan.ToModel(); Documentation = cls.Documentation; - var ntBases = cls.Bases.OfType().ToArray(); + var ntBases = cls.Bases.MaybeEnumerate().OfType().ToArray(); NamedTupleBases = ntBases.Select(b => new NamedTupleModel(b, services)).ToArray(); - Bases = cls.Bases.Except(ntBases).Select(t => t.GetPersistentQualifiedName(services)).ToArray(); + Bases = cls.Bases.MaybeEnumerate().Except(ntBases).Select(t => t.GetPersistentQualifiedName(services)).ToArray(); Methods = methods.ToArray(); Properties = properties.ToArray(); Fields = fields.ToArray(); From a2caf6a8a936c7e8e9c66a8e11d2b653af5d13dd Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 11:43:08 -0800 Subject: [PATCH 047/102] Fix one more 'missing keys' case --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index e44a306ea..35aec9cfb 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -312,6 +312,7 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, IDependencyCha foreach (var key in walker.MissingKeys.ToArray()) { lock (_syncObj) { if (_analysisEntries.TryGetValue(key, out _)) { + walker.MissingKeys = walker.MissingKeys.Remove(key); continue; } } From aa60787452a901acee826b3d9f4e66112a833873 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 12:03:51 -0800 Subject: [PATCH 048/102] Remove null declaring module from iterators --- src/Analysis/Ast/Impl/Types/LocatedMember.cs | 8 +++++--- .../Ast/Impl/Values/Collections/PythonDictionary.cs | 2 +- .../Ast/Impl/Values/Collections/PythonInstanceIterator.cs | 4 ++-- src/Analysis/Ast/Impl/Values/PythonInstance.cs | 2 +- src/Analysis/Ast/Impl/Values/PythonNone.cs | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 1d11632dd..54d732663 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -89,16 +89,18 @@ public virtual void RemoveReferences(IPythonModule module) { } internal abstract class EmptyLocatedMember : ILocatedMember { - protected EmptyLocatedMember(PythonMemberType memberType) { + protected EmptyLocatedMember(IPythonModule declaringModule, PythonMemberType memberType) { + DeclaringModule = declaringModule; MemberType = memberType; + Location = new Location(DeclaringModule); } public PythonMemberType MemberType { get; } - public IPythonModule DeclaringModule => null; + public IPythonModule DeclaringModule { get; } public LocationInfo Definition => LocationInfo.Empty; public IReadOnlyList References => Array.Empty(); public void AddReference(Location location) { } public void RemoveReferences(IPythonModule module) { } - public Location Location { get; internal set; } + public Location Location { get; } } } diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs index 53fc8175d..ae38c78f3 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs @@ -62,7 +62,7 @@ public IReadOnlyList Items _contents.TryGetValue(key, out var value) ? value : UnknownType; public override IPythonIterator GetIterator() => - Call(@"iterkeys", ArgumentSet.WithoutContext) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); + Call(@"iterkeys", ArgumentSet.WithoutContext) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule); public override IMember Index(IArgumentSet args) { if (args.Arguments.Count == 1) { diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs index 2cc207822..06f895779 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs @@ -46,8 +46,8 @@ public override IMember Call(string memberName, IArgumentSet args) { /// Empty iterator /// internal sealed class EmptyIterator : EmptyLocatedMember, IPythonIterator { - public EmptyIterator(IPythonType unknownType): base(PythonMemberType.Class) { - Type = unknownType; + public EmptyIterator(IPythonModule declaringModule): base(declaringModule, PythonMemberType.Class) { + Type = declaringModule.Interpreter.UnknownType; } public IPythonIterator GetIterator() => this; diff --git a/src/Analysis/Ast/Impl/Values/PythonInstance.cs b/src/Analysis/Ast/Impl/Values/PythonInstance.cs index 52ae31aee..43ef4969e 100644 --- a/src/Analysis/Ast/Impl/Values/PythonInstance.cs +++ b/src/Analysis/Ast/Impl/Values/PythonInstance.cs @@ -61,7 +61,7 @@ public virtual IPythonIterator GetIterator() { return new PythonInstanceIterator(instance, Type.DeclaringModule.Interpreter); } - return new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); + return new EmptyIterator(Type.DeclaringModule); } public bool Equals(IPythonInstance other) => Type?.Equals(other?.Type) == true; diff --git a/src/Analysis/Ast/Impl/Values/PythonNone.cs b/src/Analysis/Ast/Impl/Values/PythonNone.cs index 3af3360d8..78112f8f5 100644 --- a/src/Analysis/Ast/Impl/Values/PythonNone.cs +++ b/src/Analysis/Ast/Impl/Values/PythonNone.cs @@ -26,7 +26,7 @@ internal sealed class PythonNone : PythonType, IPythonInstance { public IMember Call(string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; - public IPythonIterator GetIterator() => new EmptyIterator(DeclaringModule.Interpreter.UnknownType); + public IPythonIterator GetIterator() => new EmptyIterator(DeclaringModule); public IMember Index(IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; } From d9db6a36298d9ab81dcd9cec6ad785ae78ebb62d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 15:51:07 -0800 Subject: [PATCH 049/102] Add queued cache write --- src/Caching/Impl/IO/CacheWriter.cs | 85 ++++++++++++++++++++++++++++ src/Caching/Impl/ModuleDatabase.cs | 77 ++++++------------------- src/Caching/Impl/ModuleUniqueId.cs | 13 ++++- src/Core/Impl/IO/WithRetries.cs | 46 +++++++++++++++ src/Core/Impl/Threading/TaskQueue.cs | 68 ++++++++++++++++++++++ 5 files changed, 228 insertions(+), 61 deletions(-) create mode 100644 src/Caching/Impl/IO/CacheWriter.cs create mode 100644 src/Core/Impl/IO/WithRetries.cs create mode 100644 src/Core/Impl/Threading/TaskQueue.cs diff --git a/src/Caching/Impl/IO/CacheWriter.cs b/src/Caching/Impl/IO/CacheWriter.cs new file mode 100644 index 000000000..33077fbee --- /dev/null +++ b/src/Caching/Impl/IO/CacheWriter.cs @@ -0,0 +1,85 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using LiteDB; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.Threading; + +namespace Microsoft.Python.Analysis.Caching.IO { + internal sealed class CacheWriter : IDisposable { + private readonly IFileSystem _fs; + private readonly ILogger _log; + private readonly string _cacheFolder; + private readonly TaskQueue _taskQueue; + + public CacheWriter(IFileSystem fs, ILogger log, string cacheFolder) { + _fs = fs; + _log = log; + _cacheFolder = cacheFolder; + _taskQueue = new TaskQueue(Math.Max(1, Environment.ProcessorCount / 8)); + } + + public void Dispose() { + _taskQueue.Dispose(); + } + + public Task EnqueueModel(ModuleModel model, CancellationToken cancellationToken) { + var tcs = new TaskCompletionSource(); + _taskQueue.Enqueue(() => { + try { + Write(model, cancellationToken); + tcs.SetResult(true); + } catch (OperationCanceledException) { + tcs.TrySetCanceled(); + } catch (Exception ex) when (!ex.IsCriticalException()) { + tcs.TrySetException(ex); + } + }); + return tcs.Task; + } + + private void Write(ModuleModel model, CancellationToken cancellationToken) { + if (cancellationToken.IsCancellationRequested) { + return; + } + + WithRetries.Execute(() => { + if (!_fs.DirectoryExists(_cacheFolder)) { + _fs.CreateDirectory(_cacheFolder); + } + return true; + }, $"Unable to create directory {_cacheFolder} for modules cache.", _log); + + WithRetries.Execute(() => { + if (cancellationToken.IsCancellationRequested) { + return false; + } + using (var db = new LiteDatabase(Path.Combine(_cacheFolder, $"{model.UniqueId}.db"))) { + var modules = db.GetCollection("modules"); + modules.Upsert(model); + modules.EnsureIndex(x => x.Name); + } + return true; + }, $"Unable to write analysis of {model.Name} to database.", _log); + } + } +} diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index d3bf8bf4c..6b57308d4 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -22,6 +22,7 @@ using System.Threading.Tasks; using LiteDB; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Caching.IO; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -46,6 +47,7 @@ private readonly ConcurrentDictionary _searchResults private readonly ILogger _log; private readonly IFileSystem _fs; private readonly AnalysisCachingLevel _defaultCachingLevel; + private readonly CacheWriter _cacheWriter; private AnalysisCachingLevel? _cachingLevel; public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { @@ -55,6 +57,7 @@ public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { _defaultCachingLevel = AnalysisCachingLevel.Library; var cfs = _services.GetService(); CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); + _cacheWriter = new CacheWriter(_fs, _log, CacheFolder); sm.AddService(this); } @@ -74,12 +77,6 @@ public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleT ? RestoreModule(model) : null; } - /// - /// Writes module data to the database. - /// - public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) - => Task.Run(() => StoreModuleAnalysis(analysis, cancellationToken), cancellationToken); - /// /// Determines if module analysis exists in the storage. /// @@ -105,6 +102,18 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul return false; } + public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { + var cachingLevel = GetCachingLevel(); + if (cachingLevel == AnalysisCachingLevel.None) { + return; + } + + var model = await Task.Run(() => ModuleModel.FromAnalysis(analysis, _services, cachingLevel), cancellationToken); + if (model != null && !cancellationToken.IsCancellationRequested) { + await _cacheWriter.EnqueueModel(model, cancellationToken); + } + } + internal IPythonModule RestoreModule(string moduleName, string uniqueId) { lock (_modulesLock) { if (_modulesCache.TryGetValue(uniqueId, out var m)) { @@ -126,36 +135,6 @@ private IPythonModule RestoreModule(ModuleModel model) { return dbModule; } - private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { - var cachingLevel = GetCachingLevel(); - if (cachingLevel == AnalysisCachingLevel.None) { - return; - } - - var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel); - if (model == null) { - // Caching level setting does not permit this module to be persisted. - return; - } - - WithRetries(() => { - if (!_fs.DirectoryExists(CacheFolder)) { - _fs.CreateDirectory(CacheFolder); - } - return true; - }, $"Unable to create directory {CacheFolder} for modules cache."); - - WithRetries(() => { - cancellationToken.ThrowIfCancellationRequested(); - using (var db = new LiteDatabase(Path.Combine(CacheFolder, $"{model.UniqueId}.db"))) { - var modules = db.GetCollection("modules"); - modules.Upsert(model); - modules.EnsureIndex(x => x.Name); - } - return true; - }, $"Unable to write analysis of {model.Name} to database."); - } - /// /// Locates database file based on module information. Module is identified /// by name, version, current Python interpreter version and/or hash of the @@ -205,14 +184,14 @@ private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel return true; } - model = WithRetries(() => { + model = WithRetries.Execute(() => { using (var db = new LiteDatabase(dbPath)) { var modules = db.GetCollection("modules"); var storedModel = modules.FindOne(m => m.Name == moduleName); _modelsCache[moduleName] = storedModel; return storedModel; } - }, $"Unable to locate database for module {moduleName}."); + }, $"Unable to locate database for module {moduleName}.", _log); return model != null; } @@ -221,27 +200,5 @@ private AnalysisCachingLevel GetCachingLevel() => _cachingLevel ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) ?? _defaultCachingLevel; - - private T WithRetries(Func a, string errorMessage) { - Exception ex = null; - for (var retries = 50; retries > 0; --retries) { - try { - return a(); - } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { - ex = ex1; - Thread.Sleep(10); - } catch (Exception ex2) { - ex = ex2; - break; - } - } - if (ex != null) { - _log?.Log(TraceEventType.Warning, $"{errorMessage} Exception: {ex.Message}"); - if (ex.IsCriticalException()) { - throw ex; - } - } - return default; - } } } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 19aaadce5..401e535b7 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -14,9 +14,11 @@ // permissions and limitations under the License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Modules; @@ -26,6 +28,8 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { + private static ConcurrentDictionary _idCache = new ConcurrentDictionary(); + public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { // If module is a standalone stub, permit it. Otherwise redirect to the main module // since during stub merge types from stub normally become part of the primary module. @@ -54,6 +58,11 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType } if (!string.IsNullOrEmpty(filePath) && modulePathType == PythonLibraryPathType.Site) { + var key = new AnalysisModuleKey(moduleName, filePath); + if (_idCache.TryGetValue(key, out var id)) { + return id; + } + // Module can be a submodule of a versioned package. In this case we want to use // version of the enclosing package so we have to look up the chain of folders. var moduleRootName = moduleName.Split('.')[0]; @@ -73,7 +82,9 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType if (folders.Length == 1) { var fileName = Path.GetFileNameWithoutExtension(folders[0]); var dash = fileName.IndexOf('-'); - return $"{moduleName}({fileName.Substring(dash + 1)})"; + var uniqueId = $"{moduleName}({fileName.Substring(dash + 1)})"; + _idCache[key] = uniqueId; + return uniqueId; } // Move up if nothing is found. versionFolder = Path.GetDirectoryName(versionFolder); diff --git a/src/Core/Impl/IO/WithRetries.cs b/src/Core/Impl/IO/WithRetries.cs new file mode 100644 index 000000000..41c7a7a44 --- /dev/null +++ b/src/Core/Impl/IO/WithRetries.cs @@ -0,0 +1,46 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using Microsoft.Python.Core.Logging; + +namespace Microsoft.Python.Core.IO { + public static class WithRetries { + public static T Execute(Func a, string errorMessage, ILogger log) { + Exception ex = null; + for (var retries = 50; retries > 0; --retries) { + try { + return a(); + } catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) { + ex = ex1; + Thread.Sleep(10); + } catch (Exception ex2) { + ex = ex2; + break; + } + } + if (ex != null) { + log?.Log(TraceEventType.Warning, $"{errorMessage} Exception: {ex.Message}"); + if (ex.IsCriticalException()) { + throw ex; + } + } + return default; + } + } +} diff --git a/src/Core/Impl/Threading/TaskQueue.cs b/src/Core/Impl/Threading/TaskQueue.cs new file mode 100644 index 000000000..354e57acd --- /dev/null +++ b/src/Core/Impl/Threading/TaskQueue.cs @@ -0,0 +1,68 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Microsoft.Python.Core.Threading { + // https://stackoverflow.com/questions/1656404/c-sharp-producer-consumer + // http://www.albahari.com/threading/part4.aspx#_Wait_and_Pulse + public sealed class TaskQueue : IDisposable { + private readonly object _lock = new object(); + private readonly Thread[] _workers; + private readonly Queue _queue = new Queue(); + + public TaskQueue(int maxWorkers) { + _workers = new Thread[maxWorkers]; + // Create and start a separate thread for each worker + for (var i = 0; i < _workers.Length; i++) + (_workers[i] = new Thread(Consume)).Start(); + } + + public void Dispose() { + // Enqueue one null task per worker to make each exit. + foreach (var worker in _workers) { + Enqueue(null); + } + foreach (var worker in _workers) { + worker.Join(); + } + } + + public void Enqueue(Action action) { + lock (_lock) { + _queue.Enqueue(action); + Monitor.PulseAll(_lock); + } + } + + private void Consume() { + while (true) { + Action action; + lock (_lock) { + while (_queue.Count == 0) { + Monitor.Wait(_lock); + } + action = _queue.Dequeue(); + } + if (action == null) { + break; + } + action(); + } + } + } +} From 3f9e96f1fdc01c986df7f96353a95f218ed6ef81 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 15:52:42 -0800 Subject: [PATCH 050/102] Dispose writer --- src/Caching/Impl/ModuleDatabase.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 6b57308d4..e418c1d0e 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -32,7 +31,7 @@ using Microsoft.Python.Core.Services; namespace Microsoft.Python.Analysis.Caching { - internal sealed class ModuleDatabase : IModuleDatabaseService { + internal sealed class ModuleDatabase : IModuleDatabaseService, IDisposable { private readonly object _modulesLock = new object(); private readonly Dictionary _modulesCache = new Dictionary(); @@ -200,5 +199,7 @@ private AnalysisCachingLevel GetCachingLevel() => _cachingLevel ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) ?? _defaultCachingLevel; + + public void Dispose() => _cacheWriter.Dispose(); } } From cd60912e88e1c99ac28058d77f2f73319e89b53d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 15:57:42 -0800 Subject: [PATCH 051/102] Formatting --- src/Core/Impl/Threading/TaskQueue.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Impl/Threading/TaskQueue.cs b/src/Core/Impl/Threading/TaskQueue.cs index 354e57acd..cc7144f93 100644 --- a/src/Core/Impl/Threading/TaskQueue.cs +++ b/src/Core/Impl/Threading/TaskQueue.cs @@ -28,8 +28,9 @@ public sealed class TaskQueue : IDisposable { public TaskQueue(int maxWorkers) { _workers = new Thread[maxWorkers]; // Create and start a separate thread for each worker - for (var i = 0; i < _workers.Length; i++) + for (var i = 0; i < _workers.Length; i++) { (_workers[i] = new Thread(Consume)).Start(); + } } public void Dispose() { From 72dd9c812717eb109c9b012c627b539cca10c1f5 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 17:13:28 -0800 Subject: [PATCH 052/102] Simplify unique name --- src/Caching/Impl/ModuleUniqueId.cs | 74 +----------------------------- 1 file changed, 2 insertions(+), 72 deletions(-) diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 401e535b7..974f0c738 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -13,13 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.IO; using System.Linq; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -28,8 +23,6 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { - private static ConcurrentDictionary _idCache = new ConcurrentDictionary(); - public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { // If module is a standalone stub, permit it. Otherwise redirect to the main module // since during stub merge types from stub normally become part of the primary module. @@ -57,75 +50,12 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType return null; } - if (!string.IsNullOrEmpty(filePath) && modulePathType == PythonLibraryPathType.Site) { - var key = new AnalysisModuleKey(moduleName, filePath); - if (_idCache.TryGetValue(key, out var id)) { - return id; - } - - // Module can be a submodule of a versioned package. In this case we want to use - // version of the enclosing package so we have to look up the chain of folders. - var moduleRootName = moduleName.Split('.')[0]; - var moduleFilesFolder = Path.GetDirectoryName(filePath); - var installationFolder = Path.GetDirectoryName(moduleFilesFolder); - - var versionFolder = installationFolder; - while (!string.IsNullOrEmpty(versionFolder)) { - // If module is in site-packages and is versioned, then unique id = name + version + interpreter version. - // Example: 'requests' and 'requests-2.21.0.dist-info'. - // TODO: for egg (https://github.com/microsoft/python-language-server/issues/196), consider *.egg-info - var folders = fs.GetFileSystemEntries(versionFolder, "*-*.dist-info", SearchOption.TopDirectoryOnly) - .Select(Path.GetFileName) - .Where(n => n.StartsWith(moduleRootName, StringComparison.OrdinalIgnoreCase)) // Module name can be capitalized differently. - .ToArray(); - - if (folders.Length == 1) { - var fileName = Path.GetFileNameWithoutExtension(folders[0]); - var dash = fileName.IndexOf('-'); - var uniqueId = $"{moduleName}({fileName.Substring(dash + 1)})"; - _idCache[key] = uniqueId; - return uniqueId; - } - // Move up if nothing is found. - versionFolder = Path.GetDirectoryName(versionFolder); - } - } - - var config = interpreter.Configuration; - if (moduleType == ModuleType.CompiledBuiltin || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) { - // If module is a standard library, unique id is its name + interpreter version. - return $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; - } - var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); if (parent == null) { return moduleName; } - - var hash = HashModuleFileSizes(parent); - // If all else fails, hash modules file sizes. - return $"{moduleName}.{(ulong)hash}"; - } - - private static long HashModuleFileSizes(IImportChildrenSource source) { - var hash = 0L; - var names = source.GetChildrenNames(); - foreach (var name in names) { - if (source.TryGetChildImport(name, out var child)) { - if (child is ModuleImport moduleImport) { - if (moduleImport.ModuleFileSize == 0) { - continue; // Typically test case, memory-only module. - } - hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize); - } - - if (child is IImportChildrenSource childSource) { - hash = unchecked(hash * 31 ^ HashModuleFileSizes(childSource)); - } - } - } - - return hash; + var config = interpreter.Configuration; + return $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; } private static PythonLibraryPathType GetModulePathType(string modulePath, IEnumerable libraryPaths, IFileSystem fs) { From ac490f7eecf4f200b30082bfcf164ff2fb0bd97d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 21:17:32 -0800 Subject: [PATCH 053/102] Cache unique id against multiple calls --- src/Caching/Impl/ModuleDatabase.cs | 14 ++---- src/Caching/Impl/ModuleUniqueId.cs | 80 +++++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index e418c1d0e..4a9e8acba 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -89,15 +89,11 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul return result; } - for (var retries = 50; retries > 0; --retries) { - try { - var dbPath = FindDatabaseFile(name, filePath, moduleType); - _searchResults[key] = result = !string.IsNullOrEmpty(dbPath); - return result; - } catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) { - Thread.Sleep(10); - } - } + WithRetries.Execute(() => { + var dbPath = FindDatabaseFile(name, filePath, moduleType); + _searchResults[key] = result = !string.IsNullOrEmpty(dbPath); + return result; + }, "Unable to find database file for {name}.", _log); return false; } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 974f0c738..a2eaaa7ca 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -13,8 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -23,6 +25,14 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { + private struct ModuleKey { + public string ModuleName { get; set; } + public string FilePath { get; set; } + public ModuleType ModuleType { get; set; } + } + + private static readonly ConcurrentDictionary _nameCache = new ConcurrentDictionary(); + public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { // If module is a standalone stub, permit it. Otherwise redirect to the main module // since during stub merge types from stub normally become part of the primary module. @@ -32,30 +42,84 @@ public static string GetUniqueId(this IPythonModule module, IServiceContainer se return GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); } - public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, + public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library) { - if(cachingLevel == AnalysisCachingLevel.None) { + if (cachingLevel == AnalysisCachingLevel.None) { return null; } + var key = new ModuleKey { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType }; + if (_nameCache.TryGetValue(key, out var id)) { + return id; + } + var interpreter = services.GetService(); var fs = services.GetService(); var moduleResolution = interpreter.ModuleResolution; var modulePathType = GetModulePathType(filePath, moduleResolution.LibraryPaths, fs); - switch(modulePathType) { + switch (modulePathType) { case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library: return null; case PythonLibraryPathType.StdLib when cachingLevel < AnalysisCachingLevel.System: return null; } - var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); - if (parent == null) { - return moduleName; + if (!string.IsNullOrEmpty(filePath)) { + var index = filePath.IndexOfOrdinal(".dist-info"); + if (index > 0) { + // If module is in site-packages and is versioned, then unique id = name + version + interpreter version. + // Example: 'requests' and 'requests-2.21.0.dist-info'. + // TODO: for egg (https://github.com/microsoft/python-language-server/issues/196), consider *.egg-info + var dash = filePath.IndexOf('-'); + id = $"{moduleName}({filePath.Substring(dash + 1, index - dash - 1)})"; + } + } + + if (id == null) { + var config = interpreter.Configuration; + if (moduleType == ModuleType.CompiledBuiltin || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib) { + // If module is a standard library, unique id is its name + interpreter version. + id = $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; + } + } + + if (id == null) { + var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName); + if (parent == null) { + id = moduleName; + } else { + var hash = HashModuleFileSizes(parent); + // If all else fails, hash modules file sizes. + id = $"{moduleName}.{(ulong)hash}"; + } + } + + if (id != null) { + _nameCache[key] = id; + } + return id; + } + + private static long HashModuleFileSizes(IImportChildrenSource source) { + var hash = 0L; + var names = source.GetChildrenNames(); + foreach (var name in names) { + if (source.TryGetChildImport(name, out var child)) { + if (child is ModuleImport moduleImport) { + if (moduleImport.ModuleFileSize == 0) { + continue; // Typically test case, memory-only module. + } + hash = unchecked(hash * 31 ^ moduleImport.ModuleFileSize); + } + + if (child is IImportChildrenSource childSource) { + hash = unchecked(hash * 31 ^ HashModuleFileSizes(childSource)); + } + } } - var config = interpreter.Configuration; - return $"{moduleName}({config.Version.Major}.{config.Version.Minor})"; + + return hash; } private static PythonLibraryPathType GetModulePathType(string modulePath, IEnumerable libraryPaths, IFileSystem fs) { From 053249ab692684c7fa676d09a0735f145051194a Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 11 Nov 2019 23:04:31 -0800 Subject: [PATCH 054/102] Delay writing analysis into cache until done. --- .../Analyzer/Evaluation/ExpressionEval.Scopes.cs | 2 +- src/Analysis/Ast/Impl/Values/Definitions/IScope.cs | 5 +++++ src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs | 2 +- src/Analysis/Ast/Impl/Values/Scope.cs | 8 +++++--- src/Caching/Impl/IO/CacheWriter.cs | 14 +++++++++++--- src/Caching/Impl/ModuleDatabase.cs | 2 +- src/Caching/Impl/RestoredGlobalScope.cs | 1 + src/Core/Impl/Threading/TaskQueue.cs | 10 +++++++++- 8 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index 02286c469..2a1b5787b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -163,7 +163,7 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop // node points to global scope, it is not a function or a class. scope = gs; } else { - scope = outerScope.Children.OfType().FirstOrDefault(s => s.Node == node); + scope = outerScope.GetChildScope(node) as Scope; if (scope == null) { scope = new Scope(node, outerScope, Module); outerScope.AddChildScope(scope); diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs index 7b7198783..7e2bf7695 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs @@ -48,6 +48,11 @@ public interface IScope { /// IReadOnlyList Children { get; } + /// + /// Locates child scope by name. + /// + IScope GetChildScope(ScopeStatement node); + /// /// Enumerates scopes from this one to global scope. /// diff --git a/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs b/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs index 12b50a2f2..398cf5258 100644 --- a/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs @@ -28,11 +28,11 @@ public EmptyGlobalScope(IPythonModule module) { public IPythonModule Module { get; } public string Name => string.Empty; - public PythonAst Ast => Module.Analysis.Ast; public ScopeStatement Node => Module.Analysis.Ast; public IScope OuterScope => null; public IGlobalScope GlobalScope { get; } public IReadOnlyList Children => Array.Empty(); + public IScope GetChildScope(ScopeStatement node) => null; public IEnumerable EnumerateTowardsGlobal => Enumerable.Repeat(this, 1); public IEnumerable EnumerateFromGlobal => Enumerable.Repeat(this, 1); public IVariableCollection Variables => VariableCollection.Empty; diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 2a7d967e7..2d49f8f69 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -31,7 +31,7 @@ internal class Scope : IScope { private VariableCollection _nonLocals; private VariableCollection _globals; private VariableCollection _imported; - private List _childScopes; + private Dictionary _childScopes; public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { Check.ArgumentNotNull(nameof(module), module); @@ -50,7 +50,9 @@ public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { public IScope OuterScope { get; } public IPythonModule Module { get; } - public IReadOnlyList Children => _childScopes?.ToArray() ?? Array.Empty(); + public IReadOnlyList Children => _childScopes?.Values.ToArray() ?? Array.Empty(); + public IScope GetChildScope(ScopeStatement node) => _childScopes != null && _childScopes.TryGetValue(node, out var s) ? s : null; + public IVariableCollection Variables => _variables ?? VariableCollection.Empty; public IVariableCollection NonLocals => _nonLocals ?? VariableCollection.Empty; public IVariableCollection Globals => _globals ?? VariableCollection.Empty; @@ -94,7 +96,7 @@ public void DeclareImported(string name, IMember value, Location location = defa => (_imported ?? (_imported = new VariableCollection())).DeclareVariable(name, value, VariableSource.Import, location); #endregion - internal void AddChildScope(Scope s) => (_childScopes ?? (_childScopes = new List())).Add(s); + internal void AddChildScope(Scope s) => (_childScopes ?? (_childScopes = new Dictionary()))[s.Node] = s; private VariableCollection VariableCollection => _variables ?? (_variables = new VariableCollection()); diff --git a/src/Caching/Impl/IO/CacheWriter.cs b/src/Caching/Impl/IO/CacheWriter.cs index 33077fbee..538d6e77e 100644 --- a/src/Caching/Impl/IO/CacheWriter.cs +++ b/src/Caching/Impl/IO/CacheWriter.cs @@ -18,6 +18,7 @@ using System.Threading; using System.Threading.Tasks; using LiteDB; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; @@ -30,15 +31,22 @@ internal sealed class CacheWriter : IDisposable { private readonly ILogger _log; private readonly string _cacheFolder; private readonly TaskQueue _taskQueue; + private readonly IPythonAnalyzer _analyzer; - public CacheWriter(IFileSystem fs, ILogger log, string cacheFolder) { + public CacheWriter(IPythonAnalyzer analyzer, IFileSystem fs, ILogger log, string cacheFolder) { _fs = fs; _log = log; _cacheFolder = cacheFolder; - _taskQueue = new TaskQueue(Math.Max(1, Environment.ProcessorCount / 8)); + _taskQueue = new TaskQueue(Math.Max(1, Environment.ProcessorCount / 4)); + + _analyzer = analyzer; + _analyzer.AnalysisComplete += OnAnalysisComplete; } + private void OnAnalysisComplete(object sender, AnalysisCompleteEventArgs e) => _taskQueue.ProcessQueue(); + public void Dispose() { + _analyzer.AnalysisComplete -= OnAnalysisComplete; _taskQueue.Dispose(); } @@ -53,7 +61,7 @@ public Task EnqueueModel(ModuleModel model, CancellationToken cancellationToken) } catch (Exception ex) when (!ex.IsCriticalException()) { tcs.TrySetException(ex); } - }); + }, immediate: false); return tcs.Task; } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 4a9e8acba..464dedb1f 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -56,7 +56,7 @@ public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { _defaultCachingLevel = AnalysisCachingLevel.Library; var cfs = _services.GetService(); CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}"); - _cacheWriter = new CacheWriter(_fs, _log, CacheFolder); + _cacheWriter = new CacheWriter(_services.GetService(), _fs, _log, CacheFolder); sm.AddService(this); } diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index a9db371e3..4bcf43e20 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -65,6 +65,7 @@ public void Construct(ModuleModel model, IServiceContainer services) { public ScopeStatement Node => null; public IScope OuterScope => null; public IReadOnlyList Children => Array.Empty(); + public IScope GetChildScope(ScopeStatement node) => null; public IEnumerable EnumerateTowardsGlobal => Enumerable.Empty(); public IEnumerable EnumerateFromGlobal => Enumerable.Empty(); public IVariableCollection Variables => _scopeVariables; diff --git a/src/Core/Impl/Threading/TaskQueue.cs b/src/Core/Impl/Threading/TaskQueue.cs index cc7144f93..ae6995430 100644 --- a/src/Core/Impl/Threading/TaskQueue.cs +++ b/src/Core/Impl/Threading/TaskQueue.cs @@ -43,9 +43,17 @@ public void Dispose() { } } - public void Enqueue(Action action) { + public void Enqueue(Action action, bool immediate = true) { lock (_lock) { _queue.Enqueue(action); + if (immediate) { + Monitor.PulseAll(_lock); + } + } + } + + public void ProcessQueue() { + lock (_lock) { Monitor.PulseAll(_lock); } } From 6109ac761f1c2398c5ca6b4f38826cfb28051454 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Nov 2019 13:55:07 -0800 Subject: [PATCH 055/102] Don't suppress LHS diagnostics on augmented assign --- .../UndefinedVariables/UndefinedVariablesWalker.cs | 2 -- src/Analysis/Ast/Test/AssignmentTests.cs | 9 +++++++++ src/Analysis/Ast/Test/LintUndefinedVarsTests.cs | 12 ++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 7365bf08d..42ac73bf0 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -49,9 +49,7 @@ public override bool Walk(SuiteStatement node) { HandleNonLocal(nls); break; case AugmentedAssignStatement augs: - _suppressDiagnostics = true; augs.Left?.Walk(new ExpressionWalker(this)); - _suppressDiagnostics = false; augs.Right?.Walk(new ExpressionWalker(this)); break; case AssignmentStatement asst: diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index cf3a54344..298454515 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -219,6 +219,15 @@ def __init__(self): .Which.Should().HaveMembers("abc", "y", "__class__"); } + [TestMethod, Priority(0)] + public async Task AugmentedAssignToUndefined() { + const string code = @" +x += 1 +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().NotHaveVariable("x"); + } + [TestMethod, Priority(0)] public async Task BaseInstanceVariable() { const string code = @" diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index cdee6b70f..dec8c849a 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -821,6 +821,18 @@ class Subclass(MyClass): d.Should().BeEmpty(); } + [TestMethod, Priority(0)] + public async Task AugmentedAssignToUndefined() { + const string code = @" +x += 1 +"; + var d = await LintAsync(code); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 1, 2, 2); + } + + private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); var a = Services.GetService(); From bcfc3b7246947674cbc18d7a76d2b90c105acf89 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Nov 2019 15:30:27 -0800 Subject: [PATCH 056/102] Revert "Don't suppress LHS diagnostics on augmented assign" This reverts commit 6109ac761f1c2398c5ca6b4f38826cfb28051454. --- .../UndefinedVariables/UndefinedVariablesWalker.cs | 2 ++ src/Analysis/Ast/Test/AssignmentTests.cs | 9 --------- src/Analysis/Ast/Test/LintUndefinedVarsTests.cs | 12 ------------ 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 42ac73bf0..7365bf08d 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -49,7 +49,9 @@ public override bool Walk(SuiteStatement node) { HandleNonLocal(nls); break; case AugmentedAssignStatement augs: + _suppressDiagnostics = true; augs.Left?.Walk(new ExpressionWalker(this)); + _suppressDiagnostics = false; augs.Right?.Walk(new ExpressionWalker(this)); break; case AssignmentStatement asst: diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index 298454515..cf3a54344 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -219,15 +219,6 @@ def __init__(self): .Which.Should().HaveMembers("abc", "y", "__class__"); } - [TestMethod, Priority(0)] - public async Task AugmentedAssignToUndefined() { - const string code = @" -x += 1 -"; - var analysis = await GetAnalysisAsync(code); - analysis.Should().NotHaveVariable("x"); - } - [TestMethod, Priority(0)] public async Task BaseInstanceVariable() { const string code = @" diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index dec8c849a..cdee6b70f 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -821,18 +821,6 @@ class Subclass(MyClass): d.Should().BeEmpty(); } - [TestMethod, Priority(0)] - public async Task AugmentedAssignToUndefined() { - const string code = @" -x += 1 -"; - var d = await LintAsync(code); - d.Should().HaveCount(1); - d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); - d[0].SourceSpan.Should().Be(2, 1, 2, 2); - } - - private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); var a = Services.GetService(); From e8c7a9def3e91f10ed77575433ff6427dcfea9d2 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 13 Nov 2019 09:46:29 -0800 Subject: [PATCH 057/102] Baseline updates --- .../Test/FluentAssertions/MemberAssertions.cs | 2 +- src/Caching/Impl/Models/ModuleModel.cs | 4 +-- src/Caching/Impl/ModuleFactory.cs | 4 +-- src/Caching/Impl/ModuleUniqueId.cs | 4 +++ .../Test/Files/ClassOwnDocumentation.json | 14 ++++---- src/Caching/Test/Files/NestedClasses.json | 34 +++++++++---------- src/UnitTests/Core/Impl/Baseline.cs | 1 - 7 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 784b47582..a177b0626 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -163,7 +163,7 @@ public void HaveSameMembersAs(IMember expected, bool recursive = false, string b .FailWith($"Expected '{GetName(actualContainer)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); } - // Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); + Debug.Assert(actualMemberType.MemberType == expectedMemberType.MemberType); actualMemberType.MemberType.Should().Be(expectedMemberType.MemberType, $"{expectedMemberType.Name} is {expectedMemberType.MemberType}"); #region Class comparison diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 6307b0358..8756dade9 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -163,8 +163,8 @@ public override MemberModel GetModel(string name) { if (_modelCache == null) { _modelCache = new Dictionary(); foreach (var m in GetMemberModels()) { - Debug.Assert(!_modelCache.ContainsKey(m.QualifiedName)); - _modelCache[m.QualifiedName] = m; + Debug.Assert(!_modelCache.ContainsKey(m.Name)); + _modelCache[m.Name] = m; } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index 28e6a94b2..b8ea4f3c5 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -200,8 +200,8 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { } if (member == null) { - //var containerName = mc is IPythonType t ? t.Name : ""; - //Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); + var containerName = mc is IPythonType t ? t.Name : ""; + Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); break; } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index a2eaaa7ca..91f07d45d 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -48,6 +48,10 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType return null; } + if (moduleType == ModuleType.User) { + return moduleName; // For tests where user modules are cached. + } + var key = new ModuleKey { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType }; if (_nameCache.TryGetValue(key, out var id)) { return id; diff --git a/src/Caching/Test/Files/ClassOwnDocumentation.json b/src/Caching/Test/Files/ClassOwnDocumentation.json index 0c18281c0..67b5f85b9 100644 --- a/src/Caching/Test/Files/ClassOwnDocumentation.json +++ b/src/Caching/Test/Files/ClassOwnDocumentation.json @@ -1,5 +1,5 @@ { - "UniqueId": "module.0", + "UniqueId": "module", "FilePath": null, "Documentation": "", "Functions": [], @@ -108,7 +108,7 @@ "GenericParameterValues": [], "Id": 778, "Name": "A", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:A", "IndexSpan": { "Start": 8, @@ -118,7 +118,7 @@ { "Documentation": "class A doc", "Bases": [ - "t:module:A$module.0", + "t:module:A$module", "t:object" ], "NamedTupleBases": [], @@ -129,7 +129,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B$module.0", + "Type": "t:module:B$module", "DefaultValue": null, "Kind": 0 } @@ -144,7 +144,7 @@ "Functions": [], "Id": 965872103, "Name": "__init__", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B.__init__", "IndexSpan": { "Start": 58, @@ -159,7 +159,7 @@ "GenericParameterValues": [], "Id": 779, "Name": "B", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B", "IndexSpan": { "Start": 43, @@ -204,7 +204,7 @@ } ], "FileSize": 115, - "Id": 773962309, + "Id": -2131035837, "Name": "module", "DeclaringModuleId": null, "QualifiedName": "module", diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json index 0092fc802..2a19e01a7 100644 --- a/src/Caching/Test/Files/NestedClasses.json +++ b/src/Caching/Test/Files/NestedClasses.json @@ -1,5 +1,5 @@ { - "UniqueId": "module.0", + "UniqueId": "module", "FilePath": null, "Documentation": "", "Functions": [], @@ -104,7 +104,7 @@ } }, { - "Value": "i:module:B.C$module.0", + "Value": "i:module:B.C$module", "Id": 812, "Name": "c", "DeclaringModuleId": null, @@ -129,7 +129,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:A$module.0", + "Type": "t:module:A$module", "DefaultValue": null, "Kind": 0 } @@ -144,7 +144,7 @@ "Functions": [], "Id": -1909501047, "Name": "methodA", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:A.methodA", "IndexSpan": { "Start": 33, @@ -159,7 +159,7 @@ "GenericParameterValues": [], "Id": 778, "Name": "A", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:A", "IndexSpan": { "Start": 21, @@ -179,12 +179,12 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B$module.0", + "Type": "t:module:B$module", "DefaultValue": null, "Kind": 0 } ], - "ReturnType": "i:module:B.C$module.0", + "ReturnType": "i:module:B.C$module", "Documentation": null } ], @@ -194,7 +194,7 @@ "Functions": [], "Id": 935009767, "Name": "methodB1", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B.methodB1", "IndexSpan": { "Start": 235, @@ -207,7 +207,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B$module.0", + "Type": "t:module:B$module", "DefaultValue": null, "Kind": 0 } @@ -222,7 +222,7 @@ "Functions": [], "Id": 935009768, "Name": "methodB2", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B.methodB2", "IndexSpan": { "Start": 287, @@ -255,7 +255,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B.C$module.0", + "Type": "t:module:B.C$module", "DefaultValue": null, "Kind": 0 } @@ -270,7 +270,7 @@ "Functions": [], "Id": 965872103, "Name": "__init__", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B.C.__init__", "IndexSpan": { "Start": 122, @@ -283,7 +283,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:B.C$module.0", + "Type": "t:module:B.C$module", "DefaultValue": null, "Kind": 0 } @@ -298,7 +298,7 @@ "Functions": [], "Id": -1909501045, "Name": "methodC", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B.C.methodC", "IndexSpan": { "Start": 175, @@ -322,7 +322,7 @@ "GenericParameterValues": [], "Id": 780, "Name": "C", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B.C", "IndexSpan": { "Start": 106, @@ -334,7 +334,7 @@ "GenericParameterValues": [], "Id": 779, "Name": "B", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:B", "IndexSpan": { "Start": 78, @@ -439,7 +439,7 @@ } ], "FileSize": 353, - "Id": 773962309, + "Id": -2131035837, "Name": "module", "DeclaringModuleId": null, "QualifiedName": "module", diff --git a/src/UnitTests/Core/Impl/Baseline.cs b/src/UnitTests/Core/Impl/Baseline.cs index 74e87d235..0c829a12e 100644 --- a/src/UnitTests/Core/Impl/Baseline.cs +++ b/src/UnitTests/Core/Impl/Baseline.cs @@ -93,7 +93,6 @@ public static int CompareLines(string expected, string actual, out string expect if (expectedLine == null && actualLine == null) { expectedLine = string.Empty; actualLine = string.Empty; - return 0; } From 883fd7378b477a9c81cae1ffae291912e188b74b Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 13 Nov 2019 09:48:27 -0800 Subject: [PATCH 058/102] Baselines --- src/Caching/Test/Files/SmokeTest.json | 20 ++++++++++---------- src/Caching/Test/Files/VersionHandling3.json | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json index 73e432301..335855a62 100644 --- a/src/Caching/Test/Files/SmokeTest.json +++ b/src/Caching/Test/Files/SmokeTest.json @@ -1,5 +1,5 @@ { - "UniqueId": "module.0", + "UniqueId": "module", "FilePath": null, "Documentation": "", "Functions": [ @@ -17,7 +17,7 @@ "Functions": [], "Id": 24395611, "Name": "func", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:func", "IndexSpan": { "Start": 207, @@ -126,7 +126,7 @@ } }, { - "Value": "i:module:C$module.0", + "Value": "i:module:C$module", "Id": 812, "Name": "c", "DeclaringModuleId": null, @@ -151,7 +151,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:C$module.0", + "Type": "t:module:C$module", "DefaultValue": null, "Kind": 0 } @@ -166,7 +166,7 @@ "Functions": [], "Id": 965872103, "Name": "__init__", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:C.__init__", "IndexSpan": { "Start": 45, @@ -179,7 +179,7 @@ "Parameters": [ { "Name": "self", - "Type": "t:module:C$module.0", + "Type": "t:module:C$module", "DefaultValue": null, "Kind": 0 } @@ -194,7 +194,7 @@ "Functions": [], "Id": -2139806792, "Name": "method", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:C.method", "IndexSpan": { "Start": 100, @@ -212,7 +212,7 @@ "Functions": [], "Id": 24690682, "Name": "prop", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:C.prop", "IndexSpan": { "Start": 163, @@ -243,7 +243,7 @@ "GenericParameterValues": [], "Id": 780, "Name": "C", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:C", "IndexSpan": { "Start": 21, @@ -332,7 +332,7 @@ } ], "FileSize": 243, - "Id": 773962309, + "Id": -2131035837, "Name": "module", "DeclaringModuleId": null, "QualifiedName": "module", diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json index 9db261eba..d3e874afa 100644 --- a/src/Caching/Test/Files/VersionHandling3.json +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -1,5 +1,5 @@ { - "UniqueId": "module.0", + "UniqueId": "module", "FilePath": null, "Documentation": "", "Functions": [ @@ -36,7 +36,7 @@ "Functions": [], "Id": 24395611, "Name": "func", - "DeclaringModuleId": "module.0", + "DeclaringModuleId": "module", "QualifiedName": "module:func", "IndexSpan": { "Start": 42, @@ -160,7 +160,7 @@ } ], "FileSize": 91, - "Id": 773962309, + "Id": -2131035837, "Name": "module", "DeclaringModuleId": null, "QualifiedName": "module", From 2ea16310eea7717106d116ee900f8b65f7ad9100 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 13 Nov 2019 10:45:23 -0800 Subject: [PATCH 059/102] Add option to write analysis immediately to disk for tests --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 2 +- .../Impl/Caching/Definitions/IModuleDatabaseService.cs | 9 ++++++++- src/Caching/Impl/IO/CacheWriter.cs | 4 ++-- src/Caching/Impl/ModuleDatabase.cs | 4 ++-- src/Caching/Impl/ModuleFactory.cs | 4 ++-- src/Caching/Test/RestoreTests.cs | 6 +++--- 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index c82028eb2..d7982183d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -553,7 +553,7 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainSingleNode(); - dbs?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait(); + dbs?.StoreModuleAnalysisAsync(analysis, immediate:false, _analyzerCancellationToken).DoNotWait(); return analysis; } diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index e7390198d..e04965a97 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -15,6 +15,7 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -28,7 +29,13 @@ internal interface IModuleDatabaseService: IModuleDatabaseCache { /// /// Writes module data to the database. /// - Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default); + /// Document analysis + /// + /// True if database should be written to disk immediately + /// as opposed to delaying writing until complete analysis event from the + /// + /// Cancellation token + Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default); /// /// Determines if module analysis exists in the storage. diff --git a/src/Caching/Impl/IO/CacheWriter.cs b/src/Caching/Impl/IO/CacheWriter.cs index 538d6e77e..269fba7ae 100644 --- a/src/Caching/Impl/IO/CacheWriter.cs +++ b/src/Caching/Impl/IO/CacheWriter.cs @@ -50,7 +50,7 @@ public void Dispose() { _taskQueue.Dispose(); } - public Task EnqueueModel(ModuleModel model, CancellationToken cancellationToken) { + public Task EnqueueModel(ModuleModel model, bool immediate = false, CancellationToken cancellationToken = default) { var tcs = new TaskCompletionSource(); _taskQueue.Enqueue(() => { try { @@ -61,7 +61,7 @@ public Task EnqueueModel(ModuleModel model, CancellationToken cancellationToken) } catch (Exception ex) when (!ex.IsCriticalException()) { tcs.TrySetException(ex); } - }, immediate: false); + }, immediate: immediate); return tcs.Task; } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 464dedb1f..2e51aedde 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -97,7 +97,7 @@ public bool ModuleExistsInStorage(string name, string filePath, ModuleType modul return false; } - public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { + public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default) { var cachingLevel = GetCachingLevel(); if (cachingLevel == AnalysisCachingLevel.None) { return; @@ -105,7 +105,7 @@ public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, Cancellat var model = await Task.Run(() => ModuleModel.FromAnalysis(analysis, _services, cachingLevel), cancellationToken); if (model != null && !cancellationToken.IsCancellationRequested) { - await _cacheWriter.EnqueueModel(model, cancellationToken); + await _cacheWriter.EnqueueModel(model, immediate, cancellationToken); } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index b8ea4f3c5..28e6a94b2 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -200,8 +200,8 @@ private IMember GetMember(IMember root, IEnumerable memberNames) { } if (member == null) { - var containerName = mc is IPythonType t ? t.Name : ""; - Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); + //var containerName = mc is IPythonType t ? t.Name : ""; + //Debug.Assert(member != null || EnableMissingMemberAssertions == false, $"Unable to find member {memberName} in {containerName}."); break; } diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index 43362e8b9..278a2780e 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -57,7 +57,7 @@ def func2() -> C2: ... var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); Services.AddService(dbs); - await dbs.StoreModuleAnalysisAsync(analysis2, CancellationToken.None); + await dbs.StoreModuleAnalysisAsync(analysis2, immediate: true, CancellationToken.None); await Services.GetService().ResetAnalyzer(); var doc = Services.GetService().GetDocument(analysis.Document.Uri); @@ -100,12 +100,12 @@ private async Task CreateDatabaseAsync(IPythonInterpreter interpreter) { var importedModules = interpreter.ModuleResolution.GetImportedModules(); foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { - await dbs.StoreModuleAnalysisAsync(m.Analysis, CancellationToken.None); + await dbs.StoreModuleAnalysisAsync(m.Analysis, immediate: true); } importedModules = interpreter.TypeshedResolution.GetImportedModules(); foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { - await dbs.StoreModuleAnalysisAsync(m.Analysis, CancellationToken.None); + await dbs.StoreModuleAnalysisAsync(m.Analysis, immediate: true); } } } From 471d1f76bdfa40f9b92799296ac89d504074082c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 13 Nov 2019 11:55:10 -0800 Subject: [PATCH 060/102] Workaround for named tuples naming --- .../Impl/Analyzer/Handlers/AssignmentHandler.cs | 15 ++++++++++++--- .../Typing/Definitions/ITypingNamedTupleType.cs | 11 +++++++++++ .../Typing/Types/NamedTupleType.cs | 11 ++++++++--- src/Caching/Impl/Models/ModuleModel.cs | 8 +------- src/Caching/Test/RestoreTests.cs | 3 +++ 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index af0dac748..883531979 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -15,6 +15,7 @@ using System.Diagnostics; using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; @@ -29,7 +30,7 @@ public void HandleAssignment(AssignmentStatement node, LookupOptions lookupOptio } // Filter out parenthesis expression in assignment because it makes no difference. - var lhs = node.Left.Select(s => s.RemoveParenthesis()); + var lhs = node.Left.Select(s => s.RemoveParenthesis()).ToArray(); // Note that this is handling assignments of the same value to multiple variables, // i.e. with "x = y = z = value", x/y/z are the items in lhs. If an expression looks @@ -37,8 +38,16 @@ public void HandleAssignment(AssignmentStatement node, LookupOptions lookupOptio // will be handled by AssignToExpr. var value = ExtractRhs(node.Right, lhs.FirstOrDefault(), lookupOptions); if (value != null) { - foreach (var expr in lhs) { - AssignToExpr(expr, value); + // Named tuple may get assigned to variables that have name different from the tuple itself. + // Then the name may conflict with other types in module or its persistent model. For example, + // 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...) but there is also + // 'class TokenInfo(_TokenInfo)'' so we have to use the variable name in order to avoid type conflicts. + if (value is ITypingNamedTupleType nt && lhs.Length == 1 && lhs[0] is NameExpression nex) { + nt.SetName(nex.Name); + } else { + foreach (var expr in lhs) { + AssignToExpr(expr, value); + } } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs index bc22e2a3a..00bf93bae 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/ITypingNamedTupleType.cs @@ -21,5 +21,16 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// public interface ITypingNamedTupleType : ITypingTupleType { IReadOnlyList ItemNames { get; } + /// + /// Allows setting alternative name to the tuple at the variable assignment time. + /// + /// + /// Named tuple may get assigned to variables that have name different from the tuple itself. + /// Then the name may conflict with other types in module or its persistent model. For example, + /// 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...) but there is also + /// 'class TokenInfo(_TokenInfo)'' so we have to use the variable name in order to avoid type conflicts. + /// + /// + void SetName(string name); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs index 0ba500025..98be2ad23 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/NamedTupleType.cs @@ -25,6 +25,8 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class NamedTupleType : TypingTupleType, ITypingNamedTupleType { + private string _name; + // Since named tuple operates as a new, separate type, we need to track // its location rather than delegating down to the general wrapper over // Python built-in tuple. @@ -39,7 +41,7 @@ public NamedTupleLocatedMember(Location location) : base(location) { } /// public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOnlyList itemTypes, IPythonModule declaringModule, IndexSpan indexSpan) : base(itemTypes, declaringModule, declaringModule.Interpreter) { - Name = tupleName ?? throw new ArgumentNullException(nameof(tupleName)); + _name = tupleName ?? throw new ArgumentNullException(nameof(tupleName)); ItemNames = itemNames; var typeNames = itemTypes.Select(t => t.IsUnknown() ? string.Empty : t.Name); @@ -49,11 +51,14 @@ public NamedTupleType(string tupleName, IReadOnlyList itemNames, IReadOn _locatedMember = new NamedTupleLocatedMember(new Location(declaringModule, indexSpan)); } + #region ITypingNamedTupleType public IReadOnlyList ItemNames { get; } + public void SetName(string name) => _name = name; + #endregion #region IPythonType - public override string Name { get; } - public override string QualifiedName => $"{DeclaringModule.Name}:{Name}"; // Named tuple name is a type name as class. + public override string Name => _name; + public override string QualifiedName => $"{DeclaringModule.Name}:{Name}"; public override bool IsSpecialized => true; public override string Documentation { get; } #endregion diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 8756dade9..929dccbff 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -80,7 +80,7 @@ public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceConta switch (v.Value) { case ITypingNamedTupleType nt: - namedTuples[nt.Name] = new NamedTupleModel(nt, services); + namedTuples[v.Name] = new NamedTupleModel(nt, services); continue; case IPythonFunctionType ft when ft.IsLambda(): // No need to persist lambdas. @@ -107,12 +107,6 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals continue; } break; - //case IPythonModule m: - // if (!subModules.ContainsKey(m.Name)) { - // subModules[m.Name] = new SubmoduleModel(m, services); - // continue; - // } - // break; } // Do not re-declare classes and functions as variables in the model. diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index 278a2780e..b7883c0c4 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -78,6 +78,9 @@ def func2() -> C2: ... [TestMethod, Priority(0)] public Task Os() => TestModule("os"); + [TestMethod, Priority(0)] + public Task Tokenize() => TestModule("tokenize"); + [TestMethod, Priority(0)] public Task Numpy() => TestModule("numpy"); From 8e766a20d21963835db9fbb0ce42d44b2705d108 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 13 Nov 2019 12:05:58 -0800 Subject: [PATCH 061/102] Comment --- .../Impl/Analyzer/Handlers/AssignmentHandler.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 883531979..8ee56578f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -38,16 +38,16 @@ public void HandleAssignment(AssignmentStatement node, LookupOptions lookupOptio // will be handled by AssignToExpr. var value = ExtractRhs(node.Right, lhs.FirstOrDefault(), lookupOptions); if (value != null) { - // Named tuple may get assigned to variables that have name different from the tuple itself. - // Then the name may conflict with other types in module or its persistent model. For example, - // 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...) but there is also - // 'class TokenInfo(_TokenInfo)'' so we have to use the variable name in order to avoid type conflicts. + // Named tuple may get assigned to variable that has name different from the tuple itself. + // Then the name may end up conflicting with other types in module when stub merges with + // module types. For example, 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...) + // but there is also 'class TokenInfo(_TokenInfo)' so we have to use the variable name + // in order to avoid type naming conflicts. if (value is ITypingNamedTupleType nt && lhs.Length == 1 && lhs[0] is NameExpression nex) { nt.SetName(nex.Name); - } else { - foreach (var expr in lhs) { - AssignToExpr(expr, value); - } + } + foreach (var expr in lhs) { + AssignToExpr(expr, value); } } } From 1f716e8ee2a14d558676317768231b4d71f55e34 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 14 Nov 2019 18:40:30 -0800 Subject: [PATCH 062/102] Sync builtins creation --- .../Impl/Analyzer/Definitions/IAnalyzable.cs | 6 ++++ .../Impl/Analyzer/PythonAnalyzerSession.cs | 10 +++--- .../Ast/Impl/Analyzer/PythonInterpreter.cs | 1 - .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 19 ++++++++++- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 33 ++++++++++++------- .../Resolution/MainModuleResolution.cs | 19 +++++------ 6 files changed, 59 insertions(+), 29 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 2b9182661..b13891861 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { /// @@ -30,6 +31,11 @@ internal interface IAnalyzable { /// void NotifyAnalysisBegins(); + /// + /// Performs standard analysis pass. Does not include any restoration from databases. + /// + ModuleWalker Analyze(PythonAst ast); + /// /// Notifies document that its analysis is now complete. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index caeb42dca..d52d21d29 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -527,11 +527,11 @@ private IDocumentAnalysis RestoreOrAnalyzeModule(IDependencyChainSingleNode _hiddenNames.Contains(name) ? null : base.GetMember(name); public IMember GetAnyMember(string name) => base.GetMember(name); public override IEnumerable GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray(); + #endregion + + public void Initialize() => ParseAndLogExceptions(CancellationToken.None); protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new[] { "-W", "ignore", "-B", "-E", sb }; + protected override void Parse() { } + + protected override void Analyze(PythonAst ast, int version) { + NotifyAnalysisBegins(); + var walker = Analyze(ast); + var analysis = new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + NotifyAnalysisComplete(analysis); + } + protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index d1463ef62..ba9b118bb 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,10 +22,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -305,7 +306,7 @@ public void Invalidate() { Services.GetService().InvalidateAnalysis(this); } - private void Parse() { + protected virtual void Parse() { _parseCts?.Cancel(); _parseCts = new CancellationTokenSource(); @@ -316,7 +317,7 @@ private void Parse() { _parsingTask = Task.Run(() => ParseAndLogExceptions(_linkedParseCts.Token), _linkedParseCts.Token); } - private void ParseAndLogExceptions(CancellationToken cancellationToken) { + protected void ParseAndLogExceptions(CancellationToken cancellationToken) { try { Parse(cancellationToken); } catch (Exception ex) when (!(ex is OperationCanceledException)) { @@ -325,6 +326,15 @@ private void ParseAndLogExceptions(CancellationToken cancellationToken) { } } + protected virtual void Analyze(PythonAst ast, int version) { + if (ContentState < State.Analyzing) { + ContentState = State.Analyzing; + + var analyzer = Services.GetService(); + analyzer.EnqueueDocumentForAnalysis(this, ast, version); + } + } + private void Parse(CancellationToken cancellationToken) { CollectingErrorSink sink = null; int version; @@ -345,7 +355,6 @@ private void Parse(CancellationToken cancellationToken) { } var ast = parser.ParseFile(Uri); - // Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})"); lock (_syncObj) { @@ -370,13 +379,7 @@ private void Parse(CancellationToken cancellationToken) { } NewAst?.Invoke(this, EventArgs.Empty); - - if (ContentState < State.Analyzing) { - ContentState = State.Analyzing; - - var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, ast, version); - } + Analyze(ast, version); lock (_syncObj) { _parsingTask = null; @@ -423,6 +426,14 @@ public void NotifyAnalysisBegins() { } } + public ModuleWalker Analyze(PythonAst ast) { + var eval = new ExpressionEval(Services, this, ast); + var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance); + ast.Walk(walker); + walker.Complete(); + return walker; + } + public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { lock (_syncObj) { if (analysis.Version < Analysis.Version) { diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index f138c820b..12aa1e184 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -39,6 +39,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); private readonly IUIService _ui; + private BuiltinsPythonModule _builtins; private IModuleDatabaseService _dbService; private IRunningDocumentTable _rdt; @@ -53,7 +54,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion); public ImmutableArray LibraryPaths { get; private set; } = ImmutableArray.Empty; - public IBuiltinsPythonModule BuiltinsModule { get; private set; } + public IBuiltinsPythonModule BuiltinsModule => _builtins; public IEnumerable GetImportedModules(CancellationToken cancellationToken) { foreach (var module in _specialized.Values) { @@ -185,12 +186,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation = public bool IsSpecializedModule(string fullName, string modulePath = null) => _specialized.ContainsKey(fullName); - private async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) { - var analyzer = Services.GetService(); - await analyzer.GetAnalysisAsync(BuiltinsModule, Timeout.Infinite, cancellationToken); - + private void AddBuiltinTypesToPathResolver() { Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "Builtins analysis did not complete correctly."); - // Add built-in module names var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); var value = builtinModuleNamesMember is IVariable variable ? variable.Value : builtinModuleNamesMember; @@ -222,16 +219,16 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { ReloadModulePaths(addedRoots, cancellationToken); if (!builtinsIsCreated) { - var builtinsModule = CreateBuiltinsModule(Services, Interpreter, StubCache); - BuiltinsModule = builtinsModule; - builtinsRef = new ModuleRef(builtinsModule); + _builtins = CreateBuiltinsModule(Services, Interpreter, StubCache); + builtinsRef = new ModuleRef(_builtins); + _builtins.Initialize(); } Modules[BuiltinModuleName] = builtinsRef; - await AddBuiltinTypesToPathResolverAsync(cancellationToken); + AddBuiltinTypesToPathResolver(); } - private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { + private static BuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion); var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath); return new BuiltinsPythonModule(moduleName, modulePath, services); From e3045074a9d817a08b1a080faae7a890a6229990 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 14 Nov 2019 21:22:24 -0800 Subject: [PATCH 063/102] Sync builtins --- src/LanguageServer/Impl/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 25528cdb5..85abd38b6 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 6e1a119198191015dd3d628ef54d2f4a039c331a Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Nov 2019 11:07:50 -0800 Subject: [PATCH 064/102] Usings --- src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index a49273ebf..c8caf615c 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -19,9 +19,6 @@ using System.Linq; using System.Threading; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Handlers; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Specializations; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; From 6093070396316208e112a9918002ed168a6a1bac Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Nov 2019 11:08:42 -0800 Subject: [PATCH 065/102] Debug code --- src/LanguageServer/Impl/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 85abd38b6..25528cdb5 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 0de428d44a4362bfac58b2b524f50e95fff3a88c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Nov 2019 19:44:37 -0800 Subject: [PATCH 066/102] Merge fix --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 6 ---- src/Caching/Impl/ModuleUniqueId.cs | 29 +++++++++++++++---- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index df399c64e..973f5f78a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -497,12 +497,6 @@ private void CompleteAnalysis(PythonAnalyzerEntry entry, IPythonModule module, i } private IDocumentAnalysis AnalyzeModule(IDependencyChainSingleNode node, IPythonModule module, PythonAst ast, int version) { - var analysis = _analyzer.TryRestoreCachedAnalysis(module); - if (analysis != null) { - MarkNodeWalked(node); - return analysis; - } - if (module is IAnalyzable analyzable) { var walker = analyzable.Analyze(ast); return CreateAnalysis(node, (IDocument)module, ast, version, walker); diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 91f07d45d..932c800e2 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -13,8 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; @@ -69,14 +71,31 @@ public static string GetUniqueId(string moduleName, string filePath, ModuleType return null; } - if (!string.IsNullOrEmpty(filePath)) { - var index = filePath.IndexOfOrdinal(".dist-info"); - if (index > 0) { + if (!string.IsNullOrEmpty(filePath) && modulePathType == PythonLibraryPathType.Site) { + // Module can be a submodule of a versioned package. In this case we want to use + // version of the enclosing package so we have to look up the chain of folders. + var moduleRootName = moduleName.Split('.')[0]; + var moduleFilesFolder = Path.GetDirectoryName(filePath); + var installationFolder = Path.GetDirectoryName(moduleFilesFolder); + + var versionFolder = installationFolder; + while (!string.IsNullOrEmpty(versionFolder)) { // If module is in site-packages and is versioned, then unique id = name + version + interpreter version. // Example: 'requests' and 'requests-2.21.0.dist-info'. // TODO: for egg (https://github.com/microsoft/python-language-server/issues/196), consider *.egg-info - var dash = filePath.IndexOf('-'); - id = $"{moduleName}({filePath.Substring(dash + 1, index - dash - 1)})"; + var folders = fs.GetFileSystemEntries(versionFolder, "*-*.dist-info", SearchOption.TopDirectoryOnly) + .Select(Path.GetFileName) + .Where(n => n.StartsWith(moduleRootName, StringComparison.OrdinalIgnoreCase)) // Module name can be capitalized differently. + .ToArray(); + + if (folders.Length == 1) { + var fileName = Path.GetFileNameWithoutExtension(folders[0]); + var dash = fileName.IndexOf('-'); + id = $"{moduleName}({fileName.Substring(dash + 1)})"; + break; + } + // Move up if nothing is found. + versionFolder = Path.GetDirectoryName(versionFolder); } } From b73961b54710cce968faa1031ebc427199a969af Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 20 Nov 2019 13:19:52 -0800 Subject: [PATCH 067/102] Basic stub-only mode --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 4 +--- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 14 ++++++++++++-- .../Modules/Resolution/MainModuleResolution.cs | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 201f7660c..df6fc3c1a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -79,9 +79,7 @@ private void HandleAugmentedAllAssign(AugmentedAssignStatement node) { } var rightVar = Eval.GetValueFromExpression(node.Right); - var right = rightVar as IPythonCollection; - - if (right == null) { + if (!(rightVar is IPythonCollection right)) { _allIsUsable = false; return; } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 0a8bf44e4..652dde789 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -497,8 +497,18 @@ protected virtual string LoadContent() { if (ContentState < State.Loading) { ContentState = State.Loading; try { - var code = FileSystem.ReadTextWithRetry(FilePath); - ContentState = State.Loaded; + // If this is stub, don't load content, the main module will instead use the content. + string code; + if (ModuleType == ModuleType.Stub) { + code = string.Empty; + ContentState = State.Analyzed; + } else { + var contentPath = Stub != null ? Stub.FilePath : FilePath; + code = FileSystem.ReadTextWithRetry(contentPath); + ContentState = State.Loaded; + } + //code = FileSystem.ReadTextWithRetry(FilePath); + //ContentState = State.Loaded; return code; } catch (IOException) { } catch (UnauthorizedAccessException) { } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 03913c364..b2070284f 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -106,7 +106,7 @@ protected override IPythonModule CreateModule(string name) { // If there is a stub, make sure it is loaded and attached // First check stub next to the module. if (TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { - Analyzer.InvalidateAnalysis(stub); + // Analyzer.InvalidateAnalysis(stub); } else { // If nothing found, try Typeshed. stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); From a199edddeb28fa93e45ec280b52d5a5de7bda45d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 20 Nov 2019 15:40:57 -0800 Subject: [PATCH 068/102] Add option --- .../Analyzer/Evaluation/ExpressionEval.cs | 3 ++- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 16 ++++++++------- .../Ast/Impl/Definitions/AnalysisOptions.cs | 7 +++++++ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 20 +++++++++---------- .../Impl/LanguageServer.Configuration.cs | 1 + src/LanguageServer/Impl/Program.cs | 2 +- 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 3b67d45d2..d0fa1dcab 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -46,6 +46,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); + AnalysisOptions = services.GetService()?.Options; } public GlobalScope GlobalScope { get; } @@ -56,7 +57,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public IPythonType UnknownType => Interpreter.UnknownType; public Location DefaultLocation { get; } public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule; - + public AnalysisOptions AnalysisOptions { get; } public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index b383758d1..142aefafe 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -15,7 +15,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; @@ -62,7 +61,9 @@ public override bool Walk(ClassDefinition cd) { // The variable is transient (non-user declared) hence it does not have location. // Class type is tracking locations for references and renaming. _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration); - _table.Add(new ClassEvaluator(_eval, cd)); + if (!_eval.AnalysisOptions.StubOnlyAnalysis) { + _table.Add(new ClassEvaluator(_eval, cd)); + } // Open class scope _scopes.Push(_eval.OpenScope(_eval.Module, cd, out _)); } @@ -70,8 +71,7 @@ public override bool Walk(ClassDefinition cd) { } public override void PostWalk(ClassDefinition cd) { - if (!string.IsNullOrEmpty(cd.NameExpression?.Name) && - _typeMap.ContainsKey(cd)) { + if (!string.IsNullOrEmpty(cd.NameExpression?.Name) && _typeMap.ContainsKey(cd)) { _scopes.Pop().Dispose(); } base.PostWalk(cd); @@ -138,7 +138,7 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) { private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { // Check if function exists in stubs. If so, take overload from stub // and the documentation from this actual module. - if (!_table.ReplacedByStubs.Contains(fd)) { + if (!_table.ReplacedByStubs.Contains(fd) && !_eval.AnalysisOptions.StubOnlyAnalysis) { var stubOverload = GetOverloadFromStub(fd); if (stubOverload != null) { var documentation = fd.GetDocumentation(); @@ -156,7 +156,9 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act // collection types cannot be determined as imports haven't been processed. var overload = new PythonFunctionOverload(function, fd, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); addOverload(overload); - _table.Add(new FunctionEvaluator(_eval, overload)); + if (!_eval.AnalysisOptions.StubOnlyAnalysis) { + _table.Add(new FunctionEvaluator(_eval, overload)); + } } } @@ -172,7 +174,7 @@ private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { private bool TryAddProperty(FunctionDefinition node, PythonType declaringType) { // We can't add a property to an unknown type. Fallback to a regular function for now. - // TOOD: Decouple declaring types from the property. + // TODO: Decouple declaring types from the property. if (declaringType.IsUnknown()) { return false; } diff --git a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs index dcf73ec35..232602836 100644 --- a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs +++ b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs @@ -53,5 +53,12 @@ public class AnalysisOptions { /// Defines level of caching analysis engine will maintain. /// public AnalysisCachingLevel AnalysisCachingLevel { get; set; } + + /// + /// Tells if source module should be analyzed or, if stub is present, + /// the stub becomes primary source of information on types and source + /// modules would be used only as documentation provider. + /// + public bool StubOnlyAnalysis { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 652dde789..9b002ace3 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -499,16 +499,16 @@ protected virtual string LoadContent() { try { // If this is stub, don't load content, the main module will instead use the content. string code; - if (ModuleType == ModuleType.Stub) { - code = string.Empty; - ContentState = State.Analyzed; - } else { - var contentPath = Stub != null ? Stub.FilePath : FilePath; - code = FileSystem.ReadTextWithRetry(contentPath); - ContentState = State.Loaded; - } - //code = FileSystem.ReadTextWithRetry(FilePath); - //ContentState = State.Loaded; + //if (ModuleType == ModuleType.Stub) { + // code = string.Empty; + // ContentState = State.Analyzed; + //} else { + // var contentPath = Stub != null ? Stub.FilePath : FilePath; + // code = FileSystem.ReadTextWithRetry(contentPath); + // ContentState = State.Loaded; + //} + code = FileSystem.ReadTextWithRetry(FilePath); + ContentState = State.Loaded; return code; } catch (IOException) { } catch (UnauthorizedAccessException) { } } diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index a7634fb75..14b5df6af 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -133,6 +133,7 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin optionsProvider.Options.KeepLibraryLocalVariables = GetSetting(memory, "keepLibraryLocalVariables", false); optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); optionsProvider.Options.AnalysisCachingLevel = GetAnalysisCachingLevel(analysis); + optionsProvider.Options.StubOnlyAnalysis = GetSetting(memory, "stubsOnly", false); _logger?.Log(TraceEventType.Information, Resources.AnalysisCacheLevel.FormatInvariant(optionsProvider.Options.AnalysisCachingLevel)); } diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 25528cdb5..85abd38b6 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 9ca04221afec6a9b1afd9666021e72d120d4cac2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 21 Nov 2019 16:06:40 -0800 Subject: [PATCH 069/102] Fix test code --- src/Analysis/Ast/Test/GenericsTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Analysis/Ast/Test/GenericsTests.cs b/src/Analysis/Ast/Test/GenericsTests.cs index 0f70ffd04..1ffb893c7 100644 --- a/src/Analysis/Ast/Test/GenericsTests.cs +++ b/src/Analysis/Ast/Test/GenericsTests.cs @@ -515,10 +515,13 @@ def get(self) -> _T: boxed = Box(1234) x = boxed.get() +z = boxed.v "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); analysis.Should().HaveVariable("x") .Which.Should().HaveType(BuiltinTypeId.Int); + analysis.Should().HaveVariable("z") + .Which.Should().HaveType(BuiltinTypeId.Int); } [TestMethod, Priority(0)] From 1659d2814dd3213e91d65b317582bb0eef0e4ac2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 2 Dec 2019 10:36:29 -0800 Subject: [PATCH 070/102] Limit analysis to docs and locations --- .../Ast/Impl/Analyzer/ModuleWalker.cs | 1 - .../Analyzer/Symbols/FunctionEvaluator.cs | 4 +-- .../Analyzer/Symbols/ModuleSymbolTable.cs | 2 -- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 25 ------------------- .../Impl/LanguageServer.Configuration.cs | 2 +- 5 files changed, 2 insertions(+), 32 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index df6fc3c1a..d0f05732c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -200,7 +200,6 @@ public void Complete() { _cancellationToken.ThrowIfCancellationRequested(); SymbolTable.EvaluateAll(); - SymbolTable.ReplacedByStubs.Clear(); new StubMerger(Eval).MergeStub(_stubAnalysis, _cancellationToken); if (_allIsUsable && _allReferencesCount >= 1 && GlobalScope.Variables.TryGetVariable(AllVariableName, out var variable) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 5348dbc2d..b63b3ba02 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -47,9 +47,7 @@ public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) private FunctionDefinition FunctionDefinition { get; } public override void Evaluate() { - var stub = SymbolTable.ReplacedByStubs.Contains(Target) - || _function.DeclaringModule.ModuleType == ModuleType.Stub - || Module.ModuleType == ModuleType.Specialized; + var stub = _function.DeclaringModule.ModuleType == ModuleType.Stub; using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { var returnType = TryDetermineReturnValue(); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs index 31a3b4212..97976d271 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs @@ -30,8 +30,6 @@ internal sealed class ModuleSymbolTable { private readonly Dictionary _evaluators = new Dictionary(); private readonly HashSet _processed = new HashSet(); - public HashSet ReplacedByStubs { get; } = new HashSet(); - public IEnumerable> Evaluators => _evaluators.ToArray(); public void Add(MemberEvaluator e) => _evaluators[e.Target] = e; public bool Contains(ScopeStatement node) => _evaluators.ContainsKey(node) || _processed.Contains(node); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index 142aefafe..2eefd1315 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -136,21 +136,6 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) { } private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { - // Check if function exists in stubs. If so, take overload from stub - // and the documentation from this actual module. - if (!_table.ReplacedByStubs.Contains(fd) && !_eval.AnalysisOptions.StubOnlyAnalysis) { - var stubOverload = GetOverloadFromStub(fd); - if (stubOverload != null) { - var documentation = fd.GetDocumentation(); - if (!string.IsNullOrEmpty(documentation)) { - stubOverload.SetDocumentation(documentation); - } - addOverload(stubOverload); - _table.ReplacedByStubs.Add(fd); - return; - } - } - if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. @@ -162,16 +147,6 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act } } - private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { - var t = GetMemberFromStub(node.Name).GetPythonType(); - if (t is IPythonFunctionType f) { - return f.Overloads - .OfType() - .FirstOrDefault(o => o.Parameters.Count == node.Parameters.Count(p => !p.IsPositionalOnlyMarker)); - } - return null; - } - private bool TryAddProperty(FunctionDefinition node, PythonType declaringType) { // We can't add a property to an unknown type. Fallback to a regular function for now. // TODO: Decouple declaring types from the property. diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 14b5df6af..4347cd08b 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -133,7 +133,7 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin optionsProvider.Options.KeepLibraryLocalVariables = GetSetting(memory, "keepLibraryLocalVariables", false); optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); optionsProvider.Options.AnalysisCachingLevel = GetAnalysisCachingLevel(analysis); - optionsProvider.Options.StubOnlyAnalysis = GetSetting(memory, "stubsOnly", false); + optionsProvider.Options.StubOnlyAnalysis = GetSetting(analysis, "stubsOnly", false); _logger?.Log(TraceEventType.Information, Resources.AnalysisCacheLevel.FormatInvariant(optionsProvider.Options.AnalysisCachingLevel)); } From 9fcf441e0bbebbfe7eaa496fecd67ba7bdbaa657 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 2 Dec 2019 10:40:18 -0800 Subject: [PATCH 071/102] Remove debug code --- .../Impl/LanguageServer.Configuration.cs | 2 +- src/LanguageServer/Impl/Program.cs | 58 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 4347cd08b..2da1a4a5d 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -133,7 +133,7 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin optionsProvider.Options.KeepLibraryLocalVariables = GetSetting(memory, "keepLibraryLocalVariables", false); optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); optionsProvider.Options.AnalysisCachingLevel = GetAnalysisCachingLevel(analysis); - optionsProvider.Options.StubOnlyAnalysis = GetSetting(analysis, "stubsOnly", false); + optionsProvider.Options.StubOnlyAnalysis = GetSetting(analysis, "stubsOnly", true); _logger?.Log(TraceEventType.Information, Resources.AnalysisCacheLevel.FormatInvariant(optionsProvider.Options.AnalysisCachingLevel)); } diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 85abd38b6..29ed9f88b 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; @@ -41,34 +41,34 @@ public static void Main(string[] args) { messageFormatter.JsonSerializer.ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor; messageFormatter.JsonSerializer.Converters.Add(new UriConverter()); - using (var cin = Console.OpenStandardInput()) - using (var cout = Console.OpenStandardOutput()) - using (var server = new Implementation.LanguageServer()) - using (var rpc = new LanguageServerJsonRpc(cout, cin, messageFormatter, server)) { - rpc.TraceSource.Switch.Level = SourceLevels.Error; - rpc.SynchronizationContext = new SingleThreadSynchronizationContext(); - var clientApp = new ClientApplication(rpc); - - var osp = new OSPlatform(); - services - .AddService(clientApp) - .AddService(new Logger(clientApp)) - .AddService(new UIService(clientApp)) - .AddService(new ProgressService(clientApp)) - .AddService(new TelemetryService(clientApp)) - .AddService(new IdleTimeService()) - .AddService(osp) - .AddService(new ProcessServices()) - .AddService(new FileSystem(osp)); - - services.AddService(messageFormatter.JsonSerializer); - - var token = server.Start(services, rpc); - rpc.StartListening(); - - // Wait for the "exit" request, it will terminate the process. - token.WaitHandle.WaitOne(); - } + using var cin = Console.OpenStandardInput(); + using var cout = Console.OpenStandardOutput(); + using var server = new Implementation.LanguageServer(); + using var rpc = new LanguageServerJsonRpc(cout, cin, messageFormatter, server); + + rpc.TraceSource.Switch.Level = SourceLevels.Error; + rpc.SynchronizationContext = new SingleThreadSynchronizationContext(); + var clientApp = new ClientApplication(rpc); + + var osp = new OSPlatform(); + services + .AddService(clientApp) + .AddService(new Logger(clientApp)) + .AddService(new UIService(clientApp)) + .AddService(new ProgressService(clientApp)) + .AddService(new TelemetryService(clientApp)) + .AddService(new IdleTimeService()) + .AddService(osp) + .AddService(new ProcessServices()) + .AddService(new FileSystem(osp)); + + services.AddService(messageFormatter.JsonSerializer); + + var token = server.Start(services, rpc); + rpc.StartListening(); + + // Wait for the "exit" request, it will terminate the process. + token.WaitHandle.WaitOne(); } } From 62779592b70111a540b555a8787cea9ddf70a06c Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 2 Dec 2019 12:49:06 -0800 Subject: [PATCH 072/102] Test fix --- .../Impl/Analyzer/Evaluation/ExpressionEval.cs | 17 +++++++++++++++-- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 4 ++-- src/Analysis/Ast/Test/AnalysisTestBase.cs | 7 +++++++ src/Analysis/Ast/Test/BasicTests.cs | 1 - 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index d0fa1dcab..e1ae3ae88 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -36,6 +36,7 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly Stack _openScopes = new Stack(); private readonly object _lock = new object(); private readonly List _diagnostics = new List(); + private readonly AnalysisOptions _analysisOptions; public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { Services = services ?? throw new ArgumentNullException(nameof(services)); @@ -46,7 +47,19 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); - AnalysisOptions = services.GetService()?.Options; + + var optionsOptovider = services.GetService(); + if (optionsOptovider != null) { + _analysisOptions = optionsOptovider.Options; + } else { + _analysisOptions = new AnalysisOptions { + AnalysisCachingLevel = AnalysisCachingLevel.None, + StubOnlyAnalysis = true, + KeepLibraryAst = false, + KeepLibraryLocalVariables = false, + LintingEnabled = true + }; + } } public GlobalScope GlobalScope { get; } @@ -57,7 +70,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public IPythonType UnknownType => Interpreter.UnknownType; public Location DefaultLocation { get; } public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule; - public AnalysisOptions AnalysisOptions { get; } + public bool StubOnlyAnalysis => _analysisOptions.StubOnlyAnalysis && Module.Stub != null && Module.ModuleType != ModuleType.User; public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index 2eefd1315..c9c99e2e0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -61,7 +61,7 @@ public override bool Walk(ClassDefinition cd) { // The variable is transient (non-user declared) hence it does not have location. // Class type is tracking locations for references and renaming. _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration); - if (!_eval.AnalysisOptions.StubOnlyAnalysis) { + if (!_eval.StubOnlyAnalysis) { _table.Add(new ClassEvaluator(_eval, cd)); } // Open class scope @@ -141,7 +141,7 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act // collection types cannot be determined as imports haven't been processed. var overload = new PythonFunctionOverload(function, fd, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); addOverload(overload); - if (!_eval.AnalysisOptions.StubOnlyAnalysis) { + if (!_eval.StubOnlyAnalysis) { _table.Add(new FunctionEvaluator(_eval, overload)); } } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index db915ae7f..096261ed3 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -85,6 +85,13 @@ protected async Task CreateServicesAsync(string root, Interpret sm.AddService(ds); } + var ap = Substitute.For(); + ap.Options.Returns(x => new AnalysisOptions { + AnalysisCachingLevel = AnalysisCachingLevel.None, + StubOnlyAnalysis = true + }); + sm.AddService(ap); + TestLogger.Log(TraceEventType.Information, "Create PythonAnalyzer"); CacheService.Register(sm, stubCacheFolderPath, pathCheck: false); diff --git a/src/Analysis/Ast/Test/BasicTests.cs b/src/Analysis/Ast/Test/BasicTests.cs index a469fdddc..f704f285a 100644 --- a/src/Analysis/Ast/Test/BasicTests.cs +++ b/src/Analysis/Ast/Test/BasicTests.cs @@ -18,7 +18,6 @@ using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; From c771595c66d534f398d8a03a8ec3b615c8376b63 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 2 Dec 2019 13:56:49 -0800 Subject: [PATCH 073/102] Undo accidental checkin --- src/Analysis/Ast/Test/GenericsTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Analysis/Ast/Test/GenericsTests.cs b/src/Analysis/Ast/Test/GenericsTests.cs index 1ffb893c7..0f70ffd04 100644 --- a/src/Analysis/Ast/Test/GenericsTests.cs +++ b/src/Analysis/Ast/Test/GenericsTests.cs @@ -515,13 +515,10 @@ def get(self) -> _T: boxed = Box(1234) x = boxed.get() -z = boxed.v "; var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); analysis.Should().HaveVariable("x") .Which.Should().HaveType(BuiltinTypeId.Int); - analysis.Should().HaveVariable("z") - .Which.Should().HaveType(BuiltinTypeId.Int); } [TestMethod, Priority(0)] From f93aa44ea8218cb21753f6c84cd267a4e556dcdc Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 2 Dec 2019 14:54:48 -0800 Subject: [PATCH 074/102] Add handling of exceptions to indexer --- src/Analysis/Ast/Test/AnalysisTestBase.cs | 28 ++++--- .../Impl/Indexing/IndexManager.cs | 2 +- .../Impl/Indexing/IndexParser.cs | 11 +-- .../Indexing/MostRecentDocumentSymbols.cs | 10 ++- .../Test/MissingImportCodeActionTests.cs | 79 +++++++++---------- 5 files changed, 69 insertions(+), 61 deletions(-) diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index db915ae7f..71165909a 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -37,18 +37,28 @@ using TestUtilities; namespace Microsoft.Python.Analysis.Tests { - public abstract class AnalysisTestBase { + public abstract class AnalysisTestBase: IDisposable { + private readonly CancellationTokenSource _testCts; + protected const int AnalysisTimeoutInMS = 1000 * 60; protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(AnalysisTimeoutInMS); - private TimeSpan GetAnalysisTimeout() => Debugger.IsAttached ? Timeout.InfiniteTimeSpan : AnalysisTimeout; - protected TestLogger TestLogger { get; } = new TestLogger(); protected ServiceManager Services { get; private set; } protected virtual IDiagnosticsService GetDiagnosticsService(IServiceContainer s) => null; + protected CancellationToken TestCancellationToken => _testCts.Token; + + protected AnalysisTestBase() { + _testCts = new CancellationTokenSource(Debugger.IsAttached ? Timeout.InfiniteTimeSpan : AnalysisTimeout); + } + + public void Dispose() { + _testCts.Dispose(); + } + protected ServiceManager CreateServiceManager() { Services = new ServiceManager(); @@ -163,10 +173,8 @@ protected async Task GetAnalysisAsync( TestLogger.Log(TraceEventType.Information, "Test: Analysis begin."); IDocumentAnalysis analysis; - using (var cts = new CancellationTokenSource(GetAnalysisTimeout())) { - await services.GetService().WaitForCompleteAnalysisAsync(cts.Token); - analysis = await doc.GetAnalysisAsync(-1, cts.Token); - } + await services.GetService().WaitForCompleteAnalysisAsync(TestCancellationToken); + analysis = await doc.GetAnalysisAsync(-1, TestCancellationToken); analysis.Should().NotBeNull(); TestLogger.Log(TraceEventType.Information, "Test: Analysis end."); @@ -176,10 +184,8 @@ protected async Task GetAnalysisAsync( protected async Task GetDocumentAnalysisAsync(IDocument document) { var analyzer = Services.GetService(); - using (var cts = new CancellationTokenSource(GetAnalysisTimeout())) { - await analyzer.WaitForCompleteAnalysisAsync(cts.Token); - return await document.GetAnalysisAsync(Timeout.Infinite, cts.Token); - } + await analyzer.WaitForCompleteAnalysisAsync(TestCancellationToken); + return await document.GetAnalysisAsync(Timeout.Infinite, TestCancellationToken); } } } diff --git a/src/LanguageServer/Impl/Indexing/IndexManager.cs b/src/LanguageServer/Impl/Indexing/IndexManager.cs index ea1881ca1..6e054de50 100644 --- a/src/LanguageServer/Impl/Indexing/IndexManager.cs +++ b/src/LanguageServer/Impl/Indexing/IndexManager.cs @@ -86,7 +86,7 @@ public Task IndexSnapshot(PathResolverSnapshot snapshot, CancellationToken ct = var userFiles = WorkspaceFiles(); // index library files if asked - // CreateIndices(LibraryFiles(snapshot).Except(userFiles, FileSystemInfoComparer.Instance), _libraryCodeSymbolIndex, linkedCt); + CreateIndices(LibraryFiles(snapshot).Except(userFiles, FileSystemInfoComparer.Instance), _libraryCodeSymbolIndex, linkedCt); }, linkedCt).ContinueWith(_ => linkedCts.Dispose()); } diff --git a/src/LanguageServer/Impl/Indexing/IndexParser.cs b/src/LanguageServer/Impl/Indexing/IndexParser.cs index f5da88912..03a8be120 100644 --- a/src/LanguageServer/Impl/Indexing/IndexParser.cs +++ b/src/LanguageServer/Impl/Indexing/IndexParser.cs @@ -58,12 +58,13 @@ public Task ParseAsync(string path, CancellationToken cancellationTok private async Task Parse(string path, CancellationToken parseCt) { await _semaphore.WaitAsync(parseCt); - PythonAst ast; + PythonAst ast = null; try { - using (var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - var parser = Parser.CreateParser(stream, _version); - ast = parser.ParseFile(new Uri(path)); - } + await using var stream = _fileSystem.FileOpen(path, FileMode.Open, FileAccess.Read, FileShare.Read); + var parser = Parser.CreateParser(stream, _version); + ast = parser.ParseFile(new Uri(path)); + } catch(Exception ex) when (!ex.IsCriticalException()) { + return null; } finally { _semaphore.Release(); } diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index ae9593025..acf8bcca9 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -154,10 +154,12 @@ private async Task> IndexAsync(IDocument doc, private async Task> ParseAsync(CancellationToken cancellationToken) { try { var ast = await _indexParser.ParseAsync(_path, cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); - var walker = new SymbolIndexWalker(ast, _library, cancellationToken); - ast.Walk(walker); - return walker.Symbols; + if (ast != null) { + cancellationToken.ThrowIfCancellationRequested(); + var walker = new SymbolIndexWalker(ast, _library, cancellationToken); + ast.Walk(walker); + return walker.Symbols; + } } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { Trace.TraceError(e.Message); } diff --git a/src/LanguageServer/Test/MissingImportCodeActionTests.cs b/src/LanguageServer/Test/MissingImportCodeActionTests.cs index 5bf393c2d..4adb7d653 100644 --- a/src/LanguageServer/Test/MissingImportCodeActionTests.cs +++ b/src/LanguageServer/Test/MissingImportCodeActionTests.cs @@ -32,7 +32,6 @@ using Microsoft.Python.LanguageServer.Sources; using Microsoft.Python.LanguageServer.Tests.FluentAssertions; using Microsoft.Python.Parsing.Ast; -using Microsoft.Python.Parsing.Tests; using Microsoft.Python.UnitTests.Core; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -40,8 +39,8 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class MissingImportCodeActionTests : LanguageServerTestBase { + private CancellationTokenSource _cts = new CancellationTokenSource(); public TestContext TestContext { get; set; } - public CancellationToken CancellationToken => TestContext.CancellationTokenSource.Token; [TestInitialize] public void TestInitialize() @@ -50,7 +49,7 @@ public void TestInitialize() [TestCleanup] public void Cleanup() => TestEnvironmentImpl.TestCleanup(); - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task Missing() { MarkupUtils.GetSpan(@"[|missingModule|]", out var code, out var span); @@ -58,11 +57,11 @@ public async Task Missing() { var diagnostics = GetDiagnostics(analysis, span.ToSourceSpan(analysis.Ast), MissingImportCodeActionProvider.Instance.FixableDiagnostics); diagnostics.Should().NotBeEmpty(); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); codeActions.Should().BeEmpty(); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task TopModule() { const string markup = @"{|insertionSpan:|}{|diagnostic:ntpath|}"; @@ -73,7 +72,7 @@ public async Task TopModule() { TestCodeAction(analysis.Document.Uri, codeAction, title: "import ntpath", insertionSpan, newText); } - [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0), Ignore] public async Task TopModuleFromFunctionInsertTop() { const string markup = @"{|insertionSpan:|}def TestMethod(): {|diagnostic:ntpath|}"; @@ -87,7 +86,7 @@ public async Task TopModuleFromFunctionInsertTop() { TestCodeAction(analysis.Document.Uri, codeAction, title: "import ntpath", insertionSpan, newText); } - [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0), Ignore] public async Task TopModuleLocally() { const string markup = @"def TestMethod(): {|insertionSpan:|} {|diagnostic:ntpath|}"; @@ -101,7 +100,7 @@ public async Task TopModuleLocally() { TestCodeAction(analysis.Document.Uri, codeAction, title: string.Format(Resources.ImportLocally, "import ntpath"), insertionSpan, newText); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SubModule() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:util|}", @@ -109,7 +108,7 @@ await TestCodeActionAsync( newText: "from ctypes import util" + Environment.NewLine + Environment.NewLine); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SubModuleUpdate() { await TestCodeActionAsync( @"{|insertionSpan:from ctypes import util|} @@ -118,7 +117,7 @@ await TestCodeActionAsync( newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0), Ignore] public async Task SubModuleUpdateLocally() { await TestCodeActionAsync( @"def TestMethod(): @@ -128,7 +127,7 @@ await TestCodeActionAsync( newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SubModuleFromFunctionInsertTop() { await TestCodeActionAsync( @"{|insertionSpan:|}def TestMethod(): @@ -138,7 +137,7 @@ from ctypes import util newText: "from ctypes import test" + Environment.NewLine + Environment.NewLine); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task AfterExistingImport() { await TestCodeActionAsync( @"from os import path @@ -148,7 +147,7 @@ await TestCodeActionAsync( newText: "from ctypes import util" + Environment.NewLine); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task ReplaceExistingImport() { await TestCodeActionAsync( @"from os import path @@ -160,7 +159,7 @@ import socket newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0), Ignore] public async Task AfterExistingImportLocally() { await TestCodeActionAsync( @"def TestMethod(): @@ -171,7 +170,7 @@ from os import path newText: " from ctypes import util" + Environment.NewLine); } - [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0), Ignore] public async Task ReplaceExistingImportLocally() { await TestCodeActionAsync( @"def TestMethod(): @@ -184,7 +183,7 @@ import socket newText: "from ctypes import test, util"); } - [TestMethod, Priority(0), Ignore, Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0), Ignore] public async Task CodeActionOrdering() { MarkupUtils.GetSpan(@"def TestMethod(): [|test|]", out var code, out var span); @@ -193,7 +192,7 @@ public async Task CodeActionOrdering() { var diagnostics = GetDiagnostics(analysis, span.ToSourceSpan(analysis.Ast), MissingImportCodeActionProvider.Instance.FixableDiagnostics); diagnostics.Should().NotBeEmpty(); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); var list = codeActions.Select(c => c.title).ToList(); var zipList = Enumerable.Range(0, list.Count).Zip(list); @@ -205,7 +204,7 @@ public async Task CodeActionOrdering() { maxIndexOfTopAddImports.Should().BeLessThan(minIndexOfLocalAddImports); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task PreserveComment() { await TestCodeActionAsync( @"{|insertionSpan:from os import pathconf|} # test @@ -215,7 +214,7 @@ await TestCodeActionAsync( newText: "from os import path, pathconf"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task MemberSymbol() { await TestCodeActionAsync( @"from os import path @@ -225,7 +224,7 @@ await TestCodeActionAsync( newText: "from socket import socket" + Environment.NewLine); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task NoMemberSymbol() { var markup = @"{|insertionSpan:|}{|diagnostic:socket|}"; @@ -239,7 +238,7 @@ public async Task NoMemberSymbol() { TestCodeAction(analysis.Document.Uri, codeAction, title, insertionSpan, newText); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SymbolOrdering() { var markup = @"from os import path {|insertionSpan:|} @@ -256,7 +255,7 @@ public async Task SymbolOrdering() { maxIndexOfPublicSymbol.Should().BeLessThan(minIndexOfPrivateSymbol); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SymbolOrdering2() { var markup = @"from os import path {|insertionSpan:|} @@ -275,7 +274,7 @@ public async Task SymbolOrdering2() { importedMemberIndex.Should().BeLessThan(restIndex); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SymbolOrdering3() { var markup = @"{|insertionSpan:|}{|diagnostic:pd|}"; @@ -290,7 +289,7 @@ public async Task SymbolOrdering3() { // calculate actions var diagnosticSpan = spans["diagnostic"].First().ToSourceSpan(analysis.Ast); var diagnostics = GetDiagnostics(analysis, diagnosticSpan, MissingImportCodeActionProvider.Instance.FixableDiagnostics); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); var list = codeActions.Select(c => c.title).ToList(); var zipList = Enumerable.Range(0, list.Count).Zip(list); @@ -301,7 +300,7 @@ public async Task SymbolOrdering3() { pandasIndex.Should().BeLessThan(pdIndex); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task ModuleNotReachableFromUserDocument() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:path|}", @@ -310,7 +309,7 @@ await TestCodeActionAsync( enableIndexManager: true); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SuggestAbbreviationForKnownModule() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:pandas|}", @@ -320,7 +319,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SuggestAbbreviationForKnownModule2() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:pyplot|}", @@ -330,7 +329,7 @@ await TestCodeActionAsync( relativePaths: @"matplotlib\pyplot.py"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SuggestAbbreviationForKnownModule3() { var markup = @" {|insertionSpan:from matplotlib import test|} @@ -344,7 +343,7 @@ await TestCodeActionAsync( relativePaths: new string[] { @"matplotlib\pyplot.py", @"matplotlib\test.py" }); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SuggestReverseAbbreviationForKnownModule() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:pd|}", @@ -354,7 +353,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SuggestReverseAbbreviationForKnownModule2() { await TestCodeActionAsync( @"{|insertionSpan:|}{|diagnostic:plt|}", @@ -364,7 +363,7 @@ await TestCodeActionAsync( relativePaths: @"matplotlib\pyplot.py"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task SuggestReverseAbbreviationForKnownModule3() { var markup = @" {|insertionSpan:from matplotlib import test|} @@ -378,7 +377,7 @@ await TestCodeActionAsync( relativePaths: new string[] { @"matplotlib\pyplot.py", @"matplotlib\test.py" }); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task AbbreviationConflict() { var markup = @"{|insertionSpan:|}pd = 1 @@ -392,7 +391,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task AbbreviationConflict2() { var markup = @"{|insertionSpan:|}{|diagnostic:pandas|} @@ -407,7 +406,7 @@ await TestCodeActionAsync( relativePaths: "pandas.py"); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task ContextBasedSuggestion() { var markup = @"from os import path @@ -426,7 +425,7 @@ public async Task ContextBasedSuggestion() { TestCodeAction(analysis.Document.Uri, codeAction, title, insertionSpan, newText); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task ValidToBeUsedInImport() { await TestCodeActionAsync( @"from os import path @@ -436,7 +435,7 @@ await TestCodeActionAsync( newText: "from os.path import join" + Environment.NewLine); } - [TestMethod, Priority(0), Timeout(AnalysisTimeoutInMS)] + [TestMethod, Priority(0)] public async Task Disabled() { var markup = @"from os import path [|socket|]()"; @@ -447,11 +446,11 @@ public async Task Disabled() { var diagnostics = GetDiagnostics(analysis, span.ToSourceSpan(analysis.Ast), MissingImportCodeActionProvider.Instance.FixableDiagnostics); diagnostics.Should().NotBeEmpty(); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); codeActions.Should().NotBeEmpty(); var emptyActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync( - analysis, new CodeActionSettings(null, new Dictionary() { { "addimports", false } }), diagnostics, CancellationToken); + analysis, new CodeActionSettings(null, new Dictionary() { { "addimports", false } }), diagnostics, TestCancellationToken); emptyActions.Should().BeEmpty(); } @@ -488,7 +487,7 @@ private async Task TestCodeActionAsync(string markup, string title, string newTe var insertionSpan = spans["insertionSpan"].First().ToSourceSpan(analysis.Ast); var diagnostics = GetDiagnostics(analysis, spans["diagnostic"].First().ToSourceSpan(analysis.Ast), codes); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); return (analysis, codeActions.ToArray(), insertionSpan); } @@ -522,7 +521,7 @@ private async Task TestCodeActionAsync(string markup, string title, string newTe // calculate actions var diagnosticSpan = spans["diagnostic"].First().ToSourceSpan(analysis.Ast); var diagnostics = GetDiagnostics(analysis, diagnosticSpan, MissingImportCodeActionProvider.Instance.FixableDiagnostics); - var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, CancellationToken); + var codeActions = await new QuickFixCodeActionSource(analysis.ExpressionEvaluator.Services).GetCodeActionsAsync(analysis, CodeActionSettings.Default, diagnostics, TestCancellationToken); // verify results var codeAction = codeActions.Single(c => c.title == title); From 3edce7266efe065499f0b9fc9d1094b529b22e0c Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 2 Dec 2019 15:21:49 -0800 Subject: [PATCH 075/102] Update baseline --- src/Caching/Test/Files/VersionHandling2.json | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json index ee4d02e74..9fd77e9f3 100644 --- a/src/Caching/Test/Files/VersionHandling2.json +++ b/src/Caching/Test/Files/VersionHandling2.json @@ -1,5 +1,6 @@ { "UniqueId": "module", + "FilePath": null, "Documentation": "", "Functions": [ { @@ -13,7 +14,8 @@ "Kind": 0 } ], - "ReturnType": null + "ReturnType": null, + "Documentation": null } ], "Documentation": null, @@ -22,6 +24,7 @@ "Functions": [], "Id": 24395611, "Name": "func", + "DeclaringModuleId": "module", "QualifiedName": "module:func", "IndexSpan": { "Start": 77, @@ -34,6 +37,7 @@ "Value": "t:bool", "Id": -529376420, "Name": "__debug__", + "DeclaringModuleId": null, "QualifiedName": "__debug__", "IndexSpan": { "Start": 0, @@ -44,6 +48,7 @@ "Value": "t:str", "Id": -1636005055, "Name": "__doc__", + "DeclaringModuleId": null, "QualifiedName": "__doc__", "IndexSpan": { "Start": 0, @@ -54,6 +59,7 @@ "Value": "t:str", "Id": 875442003, "Name": "__file__", + "DeclaringModuleId": null, "QualifiedName": "__file__", "IndexSpan": { "Start": 0, @@ -64,6 +70,7 @@ "Value": "t:str", "Id": 1097116834, "Name": "__name__", + "DeclaringModuleId": null, "QualifiedName": "__name__", "IndexSpan": { "Start": 0, @@ -74,6 +81,7 @@ "Value": "t:str", "Id": 75395663, "Name": "__package__", + "DeclaringModuleId": null, "QualifiedName": "__package__", "IndexSpan": { "Start": 0, @@ -84,6 +92,7 @@ "Value": "t:list", "Id": 1154586556, "Name": "__path__", + "DeclaringModuleId": null, "QualifiedName": "__path__", "IndexSpan": { "Start": 0, @@ -94,6 +103,7 @@ "Value": "t:dict", "Id": 817929997, "Name": "__dict__", + "DeclaringModuleId": null, "QualifiedName": "__dict__", "IndexSpan": { "Start": 0, @@ -104,6 +114,7 @@ "Value": "t:object", "Id": 1253875154, "Name": "__spec__", + "DeclaringModuleId": null, "QualifiedName": "__spec__", "IndexSpan": { "Start": 0, @@ -137,12 +148,9 @@ } ], "FileSize": 91, - "Imports": [], - "FromImports": [], - "StubImports": [], - "StubFromImports": [], "Id": -2131035837, "Name": "module", + "DeclaringModuleId": null, "QualifiedName": "module", "IndexSpan": null } \ No newline at end of file From e397e2acf5a2d631c8da2d0ac5a277827a3a4deb Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 2 Dec 2019 18:54:50 -0800 Subject: [PATCH 076/102] Improve synchronization of analysis sessions --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 142 +++++++----------- .../Impl/Threading/AsyncCountdownEvent.cs | 4 +- 2 files changed, 60 insertions(+), 86 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 973f5f78a..6370d60b9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -34,7 +34,6 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.OS; -using Microsoft.Python.Core.Services; using Microsoft.Python.Core.Testing; using Microsoft.Python.Parsing.Ast; @@ -54,13 +53,12 @@ internal sealed class PythonAnalyzerSession { private readonly IPythonAnalyzer _analyzer; private readonly ILogger _log; private readonly bool _forceGC; - private readonly IModuleDatabaseService _moduleDatabaseService; private readonly PathResolverSnapshot _modulesPathResolver; private readonly PathResolverSnapshot _typeshedPathResolver; + private readonly AsyncCountdownEvent _ace = new AsyncCountdownEvent(0); private State _state; private bool _isCanceled; - private int _runningTasks; public bool IsCompleted { get { @@ -97,7 +95,6 @@ public PythonAnalyzerSession(IServiceContainer services, _platformService = _services.GetService(); _analyzer = _services.GetService(); _log = _services.GetService(); - _moduleDatabaseService = _services.GetService(); _progress = progress; var interpreter = _services.GetService(); @@ -150,22 +147,19 @@ private async Task StartAsync() { } finally { stopWatch.Stop(); - bool isCanceled; - bool isFinal; lock (_syncObj) { - isCanceled = _isCanceled; _state = State.Completed; - isFinal = _walker.MissingKeys.Count == 0 && !isCanceled && remaining == 0; + var isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0; _walker = null; - } - if (!isCanceled) { - _progress.ReportRemaining(remaining); - if (isFinal) { - var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); - totalMilliseconds = Math.Round(totalMilliseconds, 2); - (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); - _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); + if (!_isCanceled) { + _progress.ReportRemaining(remaining); + if (isFinal) { + var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); + totalMilliseconds = Math.Round(totalMilliseconds, 2); + (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); + _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); + } } } } @@ -205,36 +199,29 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { var ace = new AsyncCountdownEvent(0); while ((node = await _walker.GetNextAsync(_analyzerCancellationToken)) != null) { - bool isCanceled; lock (_syncObj) { - isCanceled = _isCanceled; - } - - if (isCanceled) { - switch (node) { - case IDependencyChainLoopNode loop: - // Loop analysis is not cancellable or else small - // inner loops of a larger loop will not be analyzed - // correctly as large loop may cancel inner loop pass. - break; - case IDependencyChainSingleNode single when !single.Value.NotAnalyzed: - remaining++; - node.MoveNext(); - continue; + if (_isCanceled) { + switch (node) { + case IDependencyChainLoopNode loop: + // Loop analysis is not cancellable or else small + // inner loops of a larger loop will not be analyzed + // correctly as large loop may cancel inner loop pass. + break; + case IDependencyChainSingleNode single when !single.Value.NotAnalyzed: + remaining++; + node.MoveNext(); + continue; + } } - } - var taskLimitReached = false; - lock (_syncObj) { - _runningTasks++; - taskLimitReached = _runningTasks >= _maxTaskRunning || _walker.Remaining == 1; - } + var taskLimitReached = _ace.Count >= _maxTaskRunning || _walker.Remaining == 1; - if (taskLimitReached) { - RunAnalysis(node, stopWatch); - } else { - ace.AddOne(); - StartAnalysis(node, ace, stopWatch).DoNotWait(); + if (taskLimitReached) { + RunAnalysis(node, stopWatch); + } else { + ace.AddOne(); + StartAnalysis(node, ace, stopWatch).DoNotWait(); + } } } @@ -242,7 +229,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { lock (_syncObj) { if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { - Debug.Assert(_runningTasks == 0); + Debug.Assert(_ace.Count == 0); } else if (!_isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose) { _log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}"); } @@ -292,17 +279,12 @@ private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwat } finally { node.MoveNext(); - bool isCanceled; lock (_syncObj) { - isCanceled = _isCanceled; - } - - if (!isCanceled || loopAnalysis) { - _progress.ReportRemaining(_walker.Remaining); + if (!_isCanceled || loopAnalysis) { + _progress.ReportRemaining(_walker.Remaining); + } + ace?.Signal(); } - - Interlocked.Decrement(ref _runningTasks); - ace?.Signal(); } } @@ -504,17 +486,6 @@ private IDocumentAnalysis AnalyzeModule(IDependencyChainSingleNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { var canHaveLibraryAnalysis = false; @@ -529,33 +500,36 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainSingleNode(); - dbs?.StoreModuleAnalysisAsync(analysis, immediate:false, _analyzerCancellationToken).DoNotWait(); + var dbs = _services.GetService(); + dbs?.StoreModuleAnalysisAsync(analysis, immediate: false, _analyzerCancellationToken).DoNotWait(); - return analysis; + return analysis; + } } private void LogCompleted(IDependencyChainLoopNode node, IEnumerable modules, Stopwatch stopWatch, TimeSpan startTime) { diff --git a/src/Core/Impl/Threading/AsyncCountdownEvent.cs b/src/Core/Impl/Threading/AsyncCountdownEvent.cs index ea3b8b761..a4609c57f 100644 --- a/src/Core/Impl/Threading/AsyncCountdownEvent.cs +++ b/src/Core/Impl/Threading/AsyncCountdownEvent.cs @@ -21,7 +21,7 @@ namespace Microsoft.Python.Core { public class AsyncCountdownEvent { private readonly AsyncManualResetEvent _mre = new AsyncManualResetEvent(); - private int _count; + private long _count; public AsyncCountdownEvent(int initialCount) { if (initialCount < 0) { @@ -34,7 +34,7 @@ public AsyncCountdownEvent(int initialCount) { } } - public Task WaitAsync() => _mre.WaitAsync(); + public long Count => Interlocked.Read(ref _count); public Task WaitAsync(CancellationToken cancellationToken) => _mre.WaitAsync(cancellationToken); From 0564a2801779cb84033bc1657c245afa16ee8c22 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 2 Dec 2019 20:16:21 -0800 Subject: [PATCH 077/102] Tweak locks --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 23 ++++++++++--------- src/LanguageServer/Test/ImportsTests.cs | 3 +-- src/LanguageServer/Test/IndexParserTests.cs | 12 ---------- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 6370d60b9..96b47a2ce 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -147,20 +147,22 @@ private async Task StartAsync() { } finally { stopWatch.Stop(); + var isFinal = false; lock (_syncObj) { - _state = State.Completed; - var isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0; - _walker = null; - if (!_isCanceled) { _progress.ReportRemaining(remaining); - if (isFinal) { - var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); - totalMilliseconds = Math.Round(totalMilliseconds, 2); - (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); - _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); - } } + + _state = State.Completed; + isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0; + _walker = null; + } + + if (isFinal) { + var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); + totalMilliseconds = Math.Round(totalMilliseconds, 2); + (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); + _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); } } @@ -273,7 +275,6 @@ private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwat node.MarkWalked(); LogException(loop, exception); } - break; } } finally { diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index 516e0f062..ef60ad410 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -809,13 +809,12 @@ def bar(self): pass " + allCode; - var appCode = @" + const string appCode = @" from module1 import * A(). B(). "; - var module1Uri = TestData.GetTestSpecificUri("module1.py"); var appUri = TestData.GetTestSpecificUri("app.py"); diff --git a/src/LanguageServer/Test/IndexParserTests.cs b/src/LanguageServer/Test/IndexParserTests.cs index 32e66ec26..693bd25ff 100644 --- a/src/LanguageServer/Test/IndexParserTests.cs +++ b/src/LanguageServer/Test/IndexParserTests.cs @@ -74,18 +74,6 @@ private IReadOnlyList GetIndexSymbols(PythonAst ast) { } - [TestMethod, Priority(0)] - [ExpectedException(typeof(FileNotFoundException))] - public async Task ParseFileThatStopsExisting() { - const string testFilePath = "C:/bla.py"; - _fileSystem.FileExists(testFilePath).Returns(true); - SetFileOpen(_fileSystem, testFilePath, _ => throw new FileNotFoundException()); - - using (var indexParser = new IndexParser(_fileSystem, _pythonLanguageVersion)) { - await indexParser.ParseAsync(testFilePath); - } - } - [TestMethod, Priority(0)] public void CancelParsingAsync() { const string testFilePath = "C:/bla.py"; From cd42f7981a3451e6d1824af11fb8328c96155de3 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 2 Dec 2019 21:17:34 -0800 Subject: [PATCH 078/102] Reduce lock --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 96b47a2ce..5773203e6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -198,9 +198,9 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { IDependencyChainNode node; var remaining = 0; - var ace = new AsyncCountdownEvent(0); while ((node = await _walker.GetNextAsync(_analyzerCancellationToken)) != null) { + var taskLimitReached = false; lock (_syncObj) { if (_isCanceled) { switch (node) { @@ -216,18 +216,18 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { } } - var taskLimitReached = _ace.Count >= _maxTaskRunning || _walker.Remaining == 1; + taskLimitReached = _ace.Count >= _maxTaskRunning || _walker.Remaining == 1; + _ace.AddOne(); + } - if (taskLimitReached) { - RunAnalysis(node, stopWatch); - } else { - ace.AddOne(); - StartAnalysis(node, ace, stopWatch).DoNotWait(); - } + if (taskLimitReached) { + RunAnalysis(node, stopWatch); + } else { + StartAnalysis(node, stopWatch).DoNotWait(); } } - await ace.WaitAsync(_analyzerCancellationToken); + await _ace.WaitAsync(_analyzerCancellationToken); lock (_syncObj) { if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed)) { @@ -241,12 +241,12 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { } private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) - => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null); + => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, stopWatch), null); - private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) - => Task.Run(() => Analyze(node, ace, stopWatch)); + private Task StartAnalysis(IDependencyChainNode node, Stopwatch stopWatch) + => Task.Run(() => Analyze(node, stopWatch)); - private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { + private void Analyze(IDependencyChainNode node, Stopwatch stopWatch) { var loopAnalysis = false; try { switch (node) { @@ -284,7 +284,7 @@ private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwat if (!_isCanceled || loopAnalysis) { _progress.ReportRemaining(_walker.Remaining); } - ace?.Signal(); + _ace.Signal(); } } } From 6aee99e98099555bf60419eecf8ffd4bcc298ad3 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 2 Dec 2019 21:50:54 -0800 Subject: [PATCH 079/102] Track next session --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 3 ++- .../Impl/Analyzer/PythonAnalyzerSession.cs | 24 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 35aec9cfb..167eed09d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -20,7 +20,6 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -208,6 +207,8 @@ public IReadOnlyList LoadedModules { public event EventHandler AnalysisComplete; #endregion + internal bool HasNextSession => _nextSession != null; + internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { _analysisCompleteEvent.Set(); AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 5773203e6..f535ebb3a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -50,7 +50,7 @@ internal sealed class PythonAnalyzerSession { private readonly IDiagnosticsService _diagnosticsService; private readonly IOSPlatform _platformService; private readonly IProgressReporter _progress; - private readonly IPythonAnalyzer _analyzer; + private readonly PythonAnalyzer _analyzer; private readonly ILogger _log; private readonly bool _forceGC; private readonly PathResolverSnapshot _modulesPathResolver; @@ -93,7 +93,7 @@ public PythonAnalyzerSession(IServiceContainer services, _diagnosticsService = _services.GetService(); _platformService = _services.GetService(); - _analyzer = _services.GetService(); + _analyzer = _services.GetService(); _log = _services.GetService(); _progress = progress; @@ -104,17 +104,19 @@ public PythonAnalyzerSession(IServiceContainer services, public void Start(bool analyzeEntry) { lock (_syncObj) { + if (_state == State.Completed) { + return; + } + if (_state != State.NotStarted) { analyzeEntry = false; - } else if (_state == State.Completed) { - return; } else { _state = State.Started; } } if (analyzeEntry && _entry != null) { - Task.Run(() => AnalyzeEntry(), _analyzerCancellationToken).DoNotWait(); + Task.Run(AnalyzeEntry, _analyzerCancellationToken).DoNotWait(); } else { StartAsync().ContinueWith(_startNextSession, _analyzerCancellationToken).DoNotWait(); } @@ -144,6 +146,7 @@ private async Task StartAsync() { try { _log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started."); remaining = await AnalyzeAffectedEntriesAsync(stopWatch); + Debug.Assert(_ace.Count == 0); } finally { stopWatch.Stop(); @@ -154,14 +157,14 @@ private async Task StartAsync() { } _state = State.Completed; - isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0; + isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0 && !_analyzer.HasNextSession; _walker = null; } if (isFinal) { var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); totalMilliseconds = Math.Round(totalMilliseconds, 2); - (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); + _analyzer.RaiseAnalysisComplete(modulesCount, totalMilliseconds); _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); } } @@ -261,26 +264,21 @@ private void Analyze(IDependencyChainNode node, Stopwatch stopWatch) { node.MarkWalked(); LogException(single.Value, exception); } - break; case IDependencyChainLoopNode loop: try { loopAnalysis = true; AnalyzeLoop(loop, stopWatch); } catch (OperationCanceledException) { - //loop.Value.TryCancel(oce, _walker.Version); - //LogCanceled(single.Value.Module); } catch (Exception exception) { - //loop.Value.TrySetException(exception, _walker.Version); node.MarkWalked(); LogException(loop, exception); } break; } } finally { - node.MoveNext(); - lock (_syncObj) { + node.MoveNext(); if (!_isCanceled || loopAnalysis) { _progress.ReportRemaining(_walker.Remaining); } From cfd339ae0cb6976afe1c98b4fe947538b91e545e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 2 Dec 2019 22:15:56 -0800 Subject: [PATCH 080/102] Better next session tracking --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 8 ++++---- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 167eed09d..95ef19934 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -207,11 +207,11 @@ public IReadOnlyList LoadedModules { public event EventHandler AnalysisComplete; #endregion - internal bool HasNextSession => _nextSession != null; - internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { - _analysisCompleteEvent.Set(); - AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); + if (_nextSession == null || _currentSession?.IsCompleted == false) { + _analysisCompleteEvent.Set(); + AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); + } } private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) { diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index f535ebb3a..9b0d8d9e6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -157,7 +157,7 @@ private async Task StartAsync() { } _state = State.Completed; - isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0 && !_analyzer.HasNextSession; + isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0; _walker = null; } From a01d3097e0dec47a2d102cb89cc9463fc7f2f582 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 3 Dec 2019 13:41:06 -0800 Subject: [PATCH 081/102] Adjust timeout for slower machines --- src/LanguageServer/Test/LanguageServerTestBase.cs | 1 - src/LanguageServer/Test/MissingImportCodeActionTests.cs | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/LanguageServer/Test/LanguageServerTestBase.cs b/src/LanguageServer/Test/LanguageServerTestBase.cs index 8537b2d81..7ef8e7e93 100644 --- a/src/LanguageServer/Test/LanguageServerTestBase.cs +++ b/src/LanguageServer/Test/LanguageServerTestBase.cs @@ -26,7 +26,6 @@ public abstract class LanguageServerTestBase : AnalysisTestBase { protected static readonly ServerSettings ServerSettings = new ServerSettings(); protected override IDiagnosticsService GetDiagnosticsService(IServiceContainer s) => new DiagnosticsService(s); - protected IDiagnosticsService GetDiagnosticsService() { var ds = Services.GetService(); ds.PublishingDelay = 0; diff --git a/src/LanguageServer/Test/MissingImportCodeActionTests.cs b/src/LanguageServer/Test/MissingImportCodeActionTests.cs index 4adb7d653..ca30580a9 100644 --- a/src/LanguageServer/Test/MissingImportCodeActionTests.cs +++ b/src/LanguageServer/Test/MissingImportCodeActionTests.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis; @@ -39,12 +38,13 @@ namespace Microsoft.Python.LanguageServer.Tests { [TestClass] public class MissingImportCodeActionTests : LanguageServerTestBase { - private CancellationTokenSource _cts = new CancellationTokenSource(); public TestContext TestContext { get; set; } [TestInitialize] - public void TestInitialize() - => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + AnalysisTimeout = TimeSpan.FromMinutes(3); + } [TestCleanup] public void Cleanup() => TestEnvironmentImpl.TestCleanup(); From fd9445d7ab9ed94eb4646690830c59b7987b2ac9 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 3 Dec 2019 13:41:26 -0800 Subject: [PATCH 082/102] Timeout on slower machines --- src/Analysis/Ast/Test/AnalysisTestBase.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 71165909a..46c617ec8 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -40,9 +40,7 @@ namespace Microsoft.Python.Analysis.Tests { public abstract class AnalysisTestBase: IDisposable { private readonly CancellationTokenSource _testCts; - protected const int AnalysisTimeoutInMS = 1000 * 60; - - protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(AnalysisTimeoutInMS); + protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(1000 * 60); protected TestLogger TestLogger { get; } = new TestLogger(); protected ServiceManager Services { get; private set; } From 298e5cd4b14ec4856ccee3625f34ca68b4c6cec9 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 3 Dec 2019 14:31:56 -0800 Subject: [PATCH 083/102] Unify type member collection lock Remove readonly --- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 3 - .../Impl/Types/PythonClassType.Generics.cs | 2 +- .../Ast/Impl/Types/PythonClassType.cs | 8 +-- src/Analysis/Ast/Impl/Types/PythonType.cs | 69 ++++++++++--------- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index bd185bb20..5edae0a92 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -67,9 +67,6 @@ protected override void Analyze(PythonAst ast, int version) { protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); - foreach (var n in GetMemberNames()) { - GetMember(n).GetPythonType()?.MakeReadOnly(); - } base.OnAnalysisComplete(); } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index bb58c303a..b31a42712 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -59,7 +59,7 @@ internal partial class PythonClassType { /// B[int] inherits from A[int, str] /// public virtual IPythonType CreateSpecificType(IArgumentSet args) { - lock (_membersLock) { + lock (MembersLock) { var newGenericTypeParameters = GetTypeParameters(); var newBases = new List(); diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 3ba89e2a0..e6f805482 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -37,9 +37,7 @@ internal enum ClassDocumentationSource { Base } private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" }; - private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); - private readonly object _membersLock = new object(); private List _bases; private IReadOnlyList _mro; @@ -69,7 +67,7 @@ public PythonClassType( public override PythonMemberType MemberType => PythonMemberType.Class; public override IEnumerable GetMemberNames() { - lock (_membersLock) { + lock (MembersLock) { var names = new HashSet(Members.Keys); foreach (var m in Mro.Skip(1)) { names.UnionWith(m.GetMemberNames()); @@ -79,7 +77,7 @@ public override IEnumerable GetMemberNames() { } public override IMember GetMember(string name) { - lock (_membersLock) { + lock (MembersLock) { if (Members.TryGetValue(name, out var member)) { return member; } @@ -187,7 +185,7 @@ public override IMember Index(IPythonInstance instance, IArgumentSet args) { public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases { get { - lock (_membersLock) { + lock (MembersLock) { return _bases?.ToArray(); } } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index e7ab6c009..cdfe7fb81 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -19,22 +19,20 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{" + nameof(Name) + "}")] internal class PythonType : LocatedMember, IPythonType { - private readonly object _lock = new object(); private Dictionary _members; private BuiltinTypeId _typeId; - private bool _readonly; + protected object MembersLock { get; } = new object(); protected IReadOnlyDictionary Members => WritableMembers; private Dictionary WritableMembers => _members ?? (_members = new Dictionary()); - public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) + public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) : this(name, location, typeId) { BaseName = name ?? throw new ArgumentNullException(nameof(name)); Documentation = documentation; @@ -88,8 +86,19 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null; - public virtual IEnumerable GetMemberNames() => Members.Keys; + + public virtual IMember GetMember(string name) { + lock (MembersLock) { + return Members.TryGetValue(name, out var member) ? member : null; + } + } + + public virtual IEnumerable GetMemberNames() { + lock (MembersLock) { + return Members.Keys.ToArray(); + } + } + #endregion internal bool TrySetTypeId(BuiltinTypeId typeId) { @@ -103,29 +112,25 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { internal virtual void SetDocumentation(string documentation) => Documentation = documentation; internal void AddMembers(IEnumerable variables, bool overwrite) { - lock (_lock) { - if (!_readonly) { - foreach (var v in variables.OfType()) { - var hasMember = Members.ContainsKey(v.Name); - if (overwrite || !hasMember) { - // If variable holds function or a class, use value as member. - // If it holds an instance, use the variable itself (i.e. it is a data member). - WritableMembers[v.Name] = v.Value; - } - if (hasMember) { - v.IsClassMember = true; - } + lock (MembersLock) { + foreach (var v in variables.OfType()) { + var hasMember = Members.ContainsKey(v.Name); + if (overwrite || !hasMember) { + // If variable holds function or a class, use value as member. + // If it holds an instance, use the variable itself (i.e. it is a data member). + WritableMembers[v.Name] = v.Value; + } + if (hasMember) { + v.IsClassMember = true; } } } } internal void AddMembers(IEnumerable> members, bool overwrite) { - lock (_lock) { - if (!_readonly) { - foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { - WritableMembers[kv.Key] = kv.Value; - } + lock (MembersLock) { + foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { + WritableMembers[kv.Key] = kv.Value; } } } @@ -139,24 +144,22 @@ internal void AddMembers(IPythonClassType cls, bool overwrite) { } internal IMember AddMember(string name, IMember member, bool overwrite) { - lock (_lock) { - if (!_readonly) { - if (overwrite || !Members.ContainsKey(name)) { - WritableMembers[name] = member is IVariable v ? v.Value : member; - } + lock (MembersLock) { + if (overwrite || !Members.ContainsKey(name)) { + WritableMembers[name] = member is IVariable v ? v.Value : member; } return member; } } - internal void MakeReadOnly() { - lock (_lock) { - _readonly = true; + internal bool IsHidden => ContainsMember("__hidden__"); + + protected bool ContainsMember(string name) { + lock (MembersLock) { + return Members.ContainsKey(name); } } - internal bool IsHidden => ContainsMember("__hidden__"); - protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType; } } From ad53c29c118db263e8ab72c4dccf853f1c77a429 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 3 Dec 2019 15:24:11 -0800 Subject: [PATCH 084/102] Extend lock in invalidate --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 95ef19934..4dee8d0ca 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -119,28 +119,21 @@ public void RemoveAnalysis(IPythonModule module) { public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray analysisDependencies) { var key = new AnalysisModuleKey(module); - PythonAnalyzerEntry entry; - int version; lock (_syncObj) { - if (!_analysisEntries.TryGetValue(key, out entry)) { + if (!_analysisEntries.TryGetValue(key, out var entry)) { return; } - version = _version + 1; - } - - if (entry.Invalidate(analysisDependencies, version, out var dependencies)) { - AnalyzeDocument(key, entry, dependencies); + var version = _version + 1; + if (entry.Invalidate(analysisDependencies, version, out var dependencies)) { + AnalyzeDocument(key, entry, dependencies); + } } } public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int bufferVersion) { - PythonAnalyzerEntry entry; - AnalysisModuleKey key; - int version; - lock (_syncObj) { - entry = GetOrCreateAnalysisEntry(module, out key); - version = _version + 1; + var entry = GetOrCreateAnalysisEntry(module, out var key); + var version = _version + 1; if (entry.BufferVersion >= bufferVersion) { return; } @@ -152,10 +145,10 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int key = nonUserAsDocumentKey; entry = documentEntry; } - } - if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) { - AnalyzeDocument(key, entry, dependencies); + if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) { + AnalyzeDocument(key, entry, dependencies); + } } } @@ -220,16 +213,13 @@ private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry en _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name} ({entry.Module.ModuleType}) queued. Dependencies: {string.Join(", ", dependencies.Select(d => d.IsTypeshed ? $"{d.Name} (stub)" : d.Name))}"); var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies); - - lock (_syncObj) { - if (_version > graphVersion) { - return; - } - - _version = graphVersion; - _currentSession?.Cancel(); + if (_version > graphVersion) { + return; } + _version = graphVersion; + _currentSession?.Cancel(); + if (TryCreateSession(graphVersion, entry, out var session)) { session.Start(true); } From 234733eda02ab1040532d27a53a87c4908429dd7 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 3 Dec 2019 17:14:46 -0800 Subject: [PATCH 085/102] Revert "Extend lock in invalidate" This reverts commit ad53c29c118db263e8ab72c4dccf853f1c77a429. --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 4dee8d0ca..95ef19934 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -119,21 +119,28 @@ public void RemoveAnalysis(IPythonModule module) { public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray analysisDependencies) { var key = new AnalysisModuleKey(module); + PythonAnalyzerEntry entry; + int version; lock (_syncObj) { - if (!_analysisEntries.TryGetValue(key, out var entry)) { + if (!_analysisEntries.TryGetValue(key, out entry)) { return; } - var version = _version + 1; - if (entry.Invalidate(analysisDependencies, version, out var dependencies)) { - AnalyzeDocument(key, entry, dependencies); - } + version = _version + 1; + } + + if (entry.Invalidate(analysisDependencies, version, out var dependencies)) { + AnalyzeDocument(key, entry, dependencies); } } public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int bufferVersion) { + PythonAnalyzerEntry entry; + AnalysisModuleKey key; + int version; + lock (_syncObj) { - var entry = GetOrCreateAnalysisEntry(module, out var key); - var version = _version + 1; + entry = GetOrCreateAnalysisEntry(module, out key); + version = _version + 1; if (entry.BufferVersion >= bufferVersion) { return; } @@ -145,10 +152,10 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int key = nonUserAsDocumentKey; entry = documentEntry; } + } - if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) { - AnalyzeDocument(key, entry, dependencies); - } + if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) { + AnalyzeDocument(key, entry, dependencies); } } @@ -213,12 +220,15 @@ private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry en _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name} ({entry.Module.ModuleType}) queued. Dependencies: {string.Join(", ", dependencies.Select(d => d.IsTypeshed ? $"{d.Name} (stub)" : d.Name))}"); var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies); - if (_version > graphVersion) { - return; - } - _version = graphVersion; - _currentSession?.Cancel(); + lock (_syncObj) { + if (_version > graphVersion) { + return; + } + + _version = graphVersion; + _currentSession?.Cancel(); + } if (TryCreateSession(graphVersion, entry, out var session)) { session.Start(true); From e22f03f736b63cffbe62623f5d423ace1a43eeef Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 4 Dec 2019 11:22:51 -0800 Subject: [PATCH 086/102] Account for modules still in parsing --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 7 ++++++- src/Analysis/Ast/Test/AnalysisTestBase.cs | 6 ++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 95ef19934..79b7cd01b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -208,7 +208,12 @@ public IReadOnlyList LoadedModules { #endregion internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { - if (_nextSession == null || _currentSession?.IsCompleted == false) { + var analsysComplete = false; + lock (_syncObj) { + analsysComplete = !_analysisEntries.Values.ExcludeDefault().Any(e => e.NotAnalyzed) + && _nextSession == null && (_currentSession == null || _currentSession.IsCompleted); + } + if (analsysComplete) { _analysisCompleteEvent.Set(); AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 46c617ec8..5ad5426de 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -40,7 +40,7 @@ namespace Microsoft.Python.Analysis.Tests { public abstract class AnalysisTestBase: IDisposable { private readonly CancellationTokenSource _testCts; - protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(1000 * 60); + protected TimeSpan AnalysisTimeout { get; set; } = TimeSpan.FromMilliseconds(3000 * 60); protected TestLogger TestLogger { get; } = new TestLogger(); protected ServiceManager Services { get; private set; } @@ -167,12 +167,10 @@ protected async Task GetAnalysisAsync( var ast = await doc.GetAstAsync(CancellationToken.None); ast.Should().NotBeNull(); TestLogger.Log(TraceEventType.Information, "Test: AST end."); - TestLogger.Log(TraceEventType.Information, "Test: Analysis begin."); - IDocumentAnalysis analysis; await services.GetService().WaitForCompleteAnalysisAsync(TestCancellationToken); - analysis = await doc.GetAnalysisAsync(-1, TestCancellationToken); + var analysis = await doc.GetAnalysisAsync(-1, TestCancellationToken); analysis.Should().NotBeNull(); TestLogger.Log(TraceEventType.Information, "Test: Analysis end."); From 25fb86ac998f5c4e28797d0247c079f96fb9ebf5 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 3 Dec 2019 14:31:56 -0800 Subject: [PATCH 087/102] Unify type member collection lock Remove readonly --- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 3 - .../Impl/Types/PythonClassType.Generics.cs | 2 +- .../Ast/Impl/Types/PythonClassType.cs | 8 +-- src/Analysis/Ast/Impl/Types/PythonType.cs | 69 ++++++++++--------- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index bd185bb20..5edae0a92 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -67,9 +67,6 @@ protected override void Analyze(PythonAst ast, int version) { protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); - foreach (var n in GetMemberNames()) { - GetMember(n).GetPythonType()?.MakeReadOnly(); - } base.OnAnalysisComplete(); } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index bb58c303a..b31a42712 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -59,7 +59,7 @@ internal partial class PythonClassType { /// B[int] inherits from A[int, str] /// public virtual IPythonType CreateSpecificType(IArgumentSet args) { - lock (_membersLock) { + lock (MembersLock) { var newGenericTypeParameters = GetTypeParameters(); var newBases = new List(); diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 3ba89e2a0..e6f805482 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -37,9 +37,7 @@ internal enum ClassDocumentationSource { Base } private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" }; - private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); - private readonly object _membersLock = new object(); private List _bases; private IReadOnlyList _mro; @@ -69,7 +67,7 @@ public PythonClassType( public override PythonMemberType MemberType => PythonMemberType.Class; public override IEnumerable GetMemberNames() { - lock (_membersLock) { + lock (MembersLock) { var names = new HashSet(Members.Keys); foreach (var m in Mro.Skip(1)) { names.UnionWith(m.GetMemberNames()); @@ -79,7 +77,7 @@ public override IEnumerable GetMemberNames() { } public override IMember GetMember(string name) { - lock (_membersLock) { + lock (MembersLock) { if (Members.TryGetValue(name, out var member)) { return member; } @@ -187,7 +185,7 @@ public override IMember Index(IPythonInstance instance, IArgumentSet args) { public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases { get { - lock (_membersLock) { + lock (MembersLock) { return _bases?.ToArray(); } } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index e7ab6c009..cdfe7fb81 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -19,22 +19,20 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{" + nameof(Name) + "}")] internal class PythonType : LocatedMember, IPythonType { - private readonly object _lock = new object(); private Dictionary _members; private BuiltinTypeId _typeId; - private bool _readonly; + protected object MembersLock { get; } = new object(); protected IReadOnlyDictionary Members => WritableMembers; private Dictionary WritableMembers => _members ?? (_members = new Dictionary()); - public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) + public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) : this(name, location, typeId) { BaseName = name ?? throw new ArgumentNullException(nameof(name)); Documentation = documentation; @@ -88,8 +86,19 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null; - public virtual IEnumerable GetMemberNames() => Members.Keys; + + public virtual IMember GetMember(string name) { + lock (MembersLock) { + return Members.TryGetValue(name, out var member) ? member : null; + } + } + + public virtual IEnumerable GetMemberNames() { + lock (MembersLock) { + return Members.Keys.ToArray(); + } + } + #endregion internal bool TrySetTypeId(BuiltinTypeId typeId) { @@ -103,29 +112,25 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { internal virtual void SetDocumentation(string documentation) => Documentation = documentation; internal void AddMembers(IEnumerable variables, bool overwrite) { - lock (_lock) { - if (!_readonly) { - foreach (var v in variables.OfType()) { - var hasMember = Members.ContainsKey(v.Name); - if (overwrite || !hasMember) { - // If variable holds function or a class, use value as member. - // If it holds an instance, use the variable itself (i.e. it is a data member). - WritableMembers[v.Name] = v.Value; - } - if (hasMember) { - v.IsClassMember = true; - } + lock (MembersLock) { + foreach (var v in variables.OfType()) { + var hasMember = Members.ContainsKey(v.Name); + if (overwrite || !hasMember) { + // If variable holds function or a class, use value as member. + // If it holds an instance, use the variable itself (i.e. it is a data member). + WritableMembers[v.Name] = v.Value; + } + if (hasMember) { + v.IsClassMember = true; } } } } internal void AddMembers(IEnumerable> members, bool overwrite) { - lock (_lock) { - if (!_readonly) { - foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { - WritableMembers[kv.Key] = kv.Value; - } + lock (MembersLock) { + foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { + WritableMembers[kv.Key] = kv.Value; } } } @@ -139,24 +144,22 @@ internal void AddMembers(IPythonClassType cls, bool overwrite) { } internal IMember AddMember(string name, IMember member, bool overwrite) { - lock (_lock) { - if (!_readonly) { - if (overwrite || !Members.ContainsKey(name)) { - WritableMembers[name] = member is IVariable v ? v.Value : member; - } + lock (MembersLock) { + if (overwrite || !Members.ContainsKey(name)) { + WritableMembers[name] = member is IVariable v ? v.Value : member; } return member; } } - internal void MakeReadOnly() { - lock (_lock) { - _readonly = true; + internal bool IsHidden => ContainsMember("__hidden__"); + + protected bool ContainsMember(string name) { + lock (MembersLock) { + return Members.ContainsKey(name); } } - internal bool IsHidden => ContainsMember("__hidden__"); - protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType; } } From 2b11782dab93362bb0b110b7e39ccdb22eb857b2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 5 Dec 2019 13:23:23 -0800 Subject: [PATCH 088/102] Correct AST used in stub loop analysis --- .../Handlers/LoopImportedVariableHandler.cs | 2 +- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 18 +++++++++++------- .../Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 3 +-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs index e0a565864..788c51d27 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs @@ -106,7 +106,7 @@ public void EnsureModule(in IPythonModule module) { public ModuleWalker WalkModule(IPythonModule module, PythonAst ast) { // If module has stub, make sure it is processed too. if (module.Stub?.Analysis is EmptyAnalysis) { - WalkModule(module.Stub, module.GetAst()); + WalkModule(module.Stub, module.Stub.GetAst()); } var eval = new ExpressionEval(_services, module, ast); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 79b7cd01b..7a08efc0c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -128,7 +128,8 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray LoadedModules { #endregion internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { - var analsysComplete = false; + var analisysComplete = false; lock (_syncObj) { - analsysComplete = !_analysisEntries.Values.ExcludeDefault().Any(e => e.NotAnalyzed) - && _nextSession == null && (_currentSession == null || _currentSession.IsCompleted); + var notAnalyzed = _analysisEntries.Values.ExcludeDefault().Where(e => e.NotAnalyzed).ToArray(); + analisysComplete = notAnalyzed.Length == 0 + && _nextSession == null + && (_currentSession == null || _currentSession.IsCompleted); } - if (analsysComplete) { + if (analisysComplete) { _analysisCompleteEvent.Set(); AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); } @@ -227,7 +231,7 @@ private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry en var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsNonUserAsDocument, dependencies); lock (_syncObj) { - if (_version > graphVersion) { + if (_version >= graphVersion) { return; } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 9b0d8d9e6..5ffda908a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -341,8 +341,7 @@ private void AnalyzeLoop(IDependencyChainLoopNode loopNode, foreach (var entry in loopNode.Values) { ActivityTracker.OnEnqueueModule(entry.Module.FilePath); if (!CanUpdateAnalysis(entry, Version, out var module, out var ast)) { - _log?.Log(TraceEventType.Verbose, $"Analysis of loop canceled."); - return; + continue; } var moduleKey = new AnalysisModuleKey(module); From caf77c9a372dc8df0146a1137dd6bd2189638955 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 3 Dec 2019 14:31:56 -0800 Subject: [PATCH 089/102] Unify type member collection lock Remove readonly --- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 3 - .../Impl/Types/PythonClassType.Generics.cs | 2 +- .../Ast/Impl/Types/PythonClassType.cs | 8 +-- src/Analysis/Ast/Impl/Types/PythonType.cs | 69 ++++++++++--------- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index bd185bb20..5edae0a92 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -67,9 +67,6 @@ protected override void Analyze(PythonAst ast, int version) { protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); - foreach (var n in GetMemberNames()) { - GetMember(n).GetPythonType()?.MakeReadOnly(); - } base.OnAnalysisComplete(); } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index bb58c303a..b31a42712 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -59,7 +59,7 @@ internal partial class PythonClassType { /// B[int] inherits from A[int, str] /// public virtual IPythonType CreateSpecificType(IArgumentSet args) { - lock (_membersLock) { + lock (MembersLock) { var newGenericTypeParameters = GetTypeParameters(); var newBases = new List(); diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 3ba89e2a0..e6f805482 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -37,9 +37,7 @@ internal enum ClassDocumentationSource { Base } private static readonly string[] _classMethods = { "mro", "__dict__", @"__weakref__" }; - private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); - private readonly object _membersLock = new object(); private List _bases; private IReadOnlyList _mro; @@ -69,7 +67,7 @@ public PythonClassType( public override PythonMemberType MemberType => PythonMemberType.Class; public override IEnumerable GetMemberNames() { - lock (_membersLock) { + lock (MembersLock) { var names = new HashSet(Members.Keys); foreach (var m in Mro.Skip(1)) { names.UnionWith(m.GetMemberNames()); @@ -79,7 +77,7 @@ public override IEnumerable GetMemberNames() { } public override IMember GetMember(string name) { - lock (_membersLock) { + lock (MembersLock) { if (Members.TryGetValue(name, out var member)) { return member; } @@ -187,7 +185,7 @@ public override IMember Index(IPythonInstance instance, IArgumentSet args) { public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases { get { - lock (_membersLock) { + lock (MembersLock) { return _bases?.ToArray(); } } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index e7ab6c009..cdfe7fb81 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -19,22 +19,20 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{" + nameof(Name) + "}")] internal class PythonType : LocatedMember, IPythonType { - private readonly object _lock = new object(); private Dictionary _members; private BuiltinTypeId _typeId; - private bool _readonly; + protected object MembersLock { get; } = new object(); protected IReadOnlyDictionary Members => WritableMembers; private Dictionary WritableMembers => _members ?? (_members = new Dictionary()); - public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) + public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) : this(name, location, typeId) { BaseName = name ?? throw new ArgumentNullException(nameof(name)); Documentation = documentation; @@ -88,8 +86,19 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null; - public virtual IEnumerable GetMemberNames() => Members.Keys; + + public virtual IMember GetMember(string name) { + lock (MembersLock) { + return Members.TryGetValue(name, out var member) ? member : null; + } + } + + public virtual IEnumerable GetMemberNames() { + lock (MembersLock) { + return Members.Keys.ToArray(); + } + } + #endregion internal bool TrySetTypeId(BuiltinTypeId typeId) { @@ -103,29 +112,25 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { internal virtual void SetDocumentation(string documentation) => Documentation = documentation; internal void AddMembers(IEnumerable variables, bool overwrite) { - lock (_lock) { - if (!_readonly) { - foreach (var v in variables.OfType()) { - var hasMember = Members.ContainsKey(v.Name); - if (overwrite || !hasMember) { - // If variable holds function or a class, use value as member. - // If it holds an instance, use the variable itself (i.e. it is a data member). - WritableMembers[v.Name] = v.Value; - } - if (hasMember) { - v.IsClassMember = true; - } + lock (MembersLock) { + foreach (var v in variables.OfType()) { + var hasMember = Members.ContainsKey(v.Name); + if (overwrite || !hasMember) { + // If variable holds function or a class, use value as member. + // If it holds an instance, use the variable itself (i.e. it is a data member). + WritableMembers[v.Name] = v.Value; + } + if (hasMember) { + v.IsClassMember = true; } } } } internal void AddMembers(IEnumerable> members, bool overwrite) { - lock (_lock) { - if (!_readonly) { - foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { - WritableMembers[kv.Key] = kv.Value; - } + lock (MembersLock) { + foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { + WritableMembers[kv.Key] = kv.Value; } } } @@ -139,24 +144,22 @@ internal void AddMembers(IPythonClassType cls, bool overwrite) { } internal IMember AddMember(string name, IMember member, bool overwrite) { - lock (_lock) { - if (!_readonly) { - if (overwrite || !Members.ContainsKey(name)) { - WritableMembers[name] = member is IVariable v ? v.Value : member; - } + lock (MembersLock) { + if (overwrite || !Members.ContainsKey(name)) { + WritableMembers[name] = member is IVariable v ? v.Value : member; } return member; } } - internal void MakeReadOnly() { - lock (_lock) { - _readonly = true; + internal bool IsHidden => ContainsMember("__hidden__"); + + protected bool ContainsMember(string name) { + lock (MembersLock) { + return Members.ContainsKey(name); } } - internal bool IsHidden => ContainsMember("__hidden__"); - protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IPythonType UnknownType => DeclaringModule.Interpreter.UnknownType; } } From 10a577e3dfa7781b02853850a8ea9ae952caf352 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 5 Dec 2019 14:22:02 -0800 Subject: [PATCH 090/102] Spelling --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 7a08efc0c..45ff492f3 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -210,14 +210,14 @@ public IReadOnlyList LoadedModules { #endregion internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { - var analisysComplete = false; + var analysisComplete = false; lock (_syncObj) { var notAnalyzed = _analysisEntries.Values.ExcludeDefault().Where(e => e.NotAnalyzed).ToArray(); - analisysComplete = notAnalyzed.Length == 0 + analysisComplete = notAnalyzed.Length == 0 && _nextSession == null && (_currentSession == null || _currentSession.IsCompleted); } - if (analisysComplete) { + if (analysisComplete) { _analysisCompleteEvent.Set(); AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); } From 4dbc1b19724321e502258d6c74593c5771afdeec Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 5 Dec 2019 15:44:16 -0800 Subject: [PATCH 091/102] Null check --- .../Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs index 788c51d27..8a4930bde 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs @@ -105,7 +105,7 @@ public void EnsureModule(in IPythonModule module) { public ModuleWalker WalkModule(IPythonModule module, PythonAst ast) { // If module has stub, make sure it is processed too. - if (module.Stub?.Analysis is EmptyAnalysis) { + if (module.Stub?.Analysis is EmptyAnalysis && module.Stub.GetAst() != null) { WalkModule(module.Stub, module.Stub.GetAst()); } From 026b40a295cf8bd81dd9b054a7f2a31ba660ab5a Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 5 Dec 2019 15:47:23 -0800 Subject: [PATCH 092/102] Comment out services dispose for test stability since when test completes presions analysis session that was canceled may still be running. --- src/Core/Impl/Services/ServiceManager.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Core/Impl/Services/ServiceManager.cs b/src/Core/Impl/Services/ServiceManager.cs index eeaede73b..2cc43fbfe 100644 --- a/src/Core/Impl/Services/ServiceManager.cs +++ b/src/Core/Impl/Services/ServiceManager.cs @@ -66,13 +66,13 @@ public IServiceManager AddService(Func factory) where T /// Service type /// Service instance or null if it doesn't exist public T GetService(Type type = null) where T : class { - if (_disposeToken.IsDisposed) { + //if (_disposeToken.IsDisposed) { // Do not throw. When editor text buffer is closed, the associated service manager // is disposed. However, some actions may still hold on the text buffer reference // and actually determine if buffer is closed by checking if editor document // is still attached as a service. - return null; - } + //return null; + //} type = type ?? typeof(T); if (!_s.TryGetValue(type, out var value)) { @@ -98,13 +98,13 @@ private object CheckDisposed(object service) { #region IDisposable public void Dispose() { if (_disposeToken.TryMarkDisposed()) { - foreach (var service in _s.Values) { - if (service is Lazy lazy && lazy.IsValueCreated) { - (lazy.Value as IDisposable)?.Dispose(); - } else { - (service as IDisposable)?.Dispose(); - } - } + //foreach (var service in _s.Values) { + // if (service is Lazy lazy && lazy.IsValueCreated) { + // (lazy.Value as IDisposable)?.Dispose(); + // } else { + // (service as IDisposable)?.Dispose(); + // } + //} } } #endregion From e63a4acfa523d7a3e7e8532d32bf6e503dcd8ed8 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 5 Dec 2019 15:48:44 -0800 Subject: [PATCH 093/102] Remove another dispose --- src/Core/Impl/Services/ServiceManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Impl/Services/ServiceManager.cs b/src/Core/Impl/Services/ServiceManager.cs index 2cc43fbfe..6648252c1 100644 --- a/src/Core/Impl/Services/ServiceManager.cs +++ b/src/Core/Impl/Services/ServiceManager.cs @@ -79,7 +79,7 @@ public T GetService(Type type = null) where T : class { value = _s.FirstOrDefault(kvp => type.GetTypeInfo().IsAssignableFrom(kvp.Key)).Value; } - return (T)CheckDisposed(value as T ?? (value as Lazy)?.Value); + return value as T ?? (value as Lazy)?.Value as T; } public void RemoveService(object service) => _s.TryRemove(service.GetType(), out var dummy); From 337ff52945c38cb572f1de2dca49b592d222e731 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 5 Dec 2019 17:16:56 -0800 Subject: [PATCH 094/102] Old code in comments --- src/Core/Impl/Services/ServiceManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/Impl/Services/ServiceManager.cs b/src/Core/Impl/Services/ServiceManager.cs index 6648252c1..4c8c1140f 100644 --- a/src/Core/Impl/Services/ServiceManager.cs +++ b/src/Core/Impl/Services/ServiceManager.cs @@ -79,6 +79,7 @@ public T GetService(Type type = null) where T : class { value = _s.FirstOrDefault(kvp => type.GetTypeInfo().IsAssignableFrom(kvp.Key)).Value; } + //return (T)CheckDisposed(value as T ?? (value as Lazy)?.Value); return value as T ?? (value as Lazy)?.Value as T; } From 2c7e7292f2224869fef18d691722d1133303233d Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 16 Dec 2019 10:53:32 -0800 Subject: [PATCH 095/102] Merge issues --- .../Ast/Impl/Analyzer/Symbols/SymbolCollector.cs | 15 --------------- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 14 ++------------ 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index a36ac631e..c9c99e2e0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -136,21 +136,6 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) { } private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { - // Check if function exists in stubs. If so, take overload from stub - // and the documentation from this actual module. - if (!_table.ReplacedByStubs.Contains(fd)) { - var stubOverload = GetOverloadFromStub(fd); - if (stubOverload != null) { - var documentation = fd.GetDocumentation(); - if (!string.IsNullOrEmpty(documentation)) { - stubOverload.SetDocumentation(documentation); - } - addOverload(stubOverload); - _table.ReplacedByStubs.Add(fd); - return; - } - } - if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 4374bf96d..dba50e7e7 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -487,18 +487,8 @@ protected virtual string LoadContent() { if (ModuleState < ModuleState.Loading) { ModuleState = ModuleState.Loading; try { - // If this is stub, don't load content, the main module will instead use the content. - string code; - //if (ModuleType == ModuleType.Stub) { - // code = string.Empty; - // ContentState = State.Analyzed; - //} else { - // var contentPath = Stub != null ? Stub.FilePath : FilePath; - // code = FileSystem.ReadTextWithRetry(contentPath); - // ContentState = State.Loaded; - //} - code = FileSystem.ReadTextWithRetry(FilePath); - ModuleState = State.Loaded; + var code = FileSystem.ReadTextWithRetry(FilePath); + ModuleState = ModuleState.Loaded; return code; } catch (IOException) { } catch (UnauthorizedAccessException) { } } From 18b619e7d62d67cfcedaca1b3bd65eb1d64caf32 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 16 Dec 2019 14:55:06 -0800 Subject: [PATCH 096/102] Uncomment --- .../Ast/Impl/Modules/Resolution/MainModuleResolution.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index b2070284f..03913c364 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -106,7 +106,7 @@ protected override IPythonModule CreateModule(string name) { // If there is a stub, make sure it is loaded and attached // First check stub next to the module. if (TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { - // Analyzer.InvalidateAnalysis(stub); + Analyzer.InvalidateAnalysis(stub); } else { // If nothing found, try Typeshed. stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); From 1c59d01bbacd0b1923d7b9ba4b5e8f1bc4b083de Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 16 Dec 2019 15:09:04 -0800 Subject: [PATCH 097/102] Fix test analysis options --- src/Analysis/Ast/Test/AnalysisTestBase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index f5bff3586..28a29e6bf 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -96,7 +96,8 @@ protected async Task CreateServicesAsync(string root, Interpret var ap = Substitute.For(); ap.Options.Returns(x => new AnalysisOptions { AnalysisCachingLevel = AnalysisCachingLevel.None, - StubOnlyAnalysis = true + StubOnlyAnalysis = true, + LintingEnabled = true }); sm.AddService(ap); From bf030c9a42664b10acbbcbde4570bc1ac0d2d8eb Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 16 Dec 2019 15:28:33 -0800 Subject: [PATCH 098/102] Undo too aggressive optimization --- .../Impl/Analyzer/Symbols/ClassEvaluator.cs | 59 ++++++++++--------- .../Analyzer/Symbols/FunctionEvaluator.cs | 4 ++ .../Impl/Analyzer/Symbols/SymbolCollector.cs | 4 +- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index cfc515212..422494537 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -72,36 +72,41 @@ private void ProcessClassBody() { SymbolTable.Evaluate(b.ClassDefinition); } - // Process imports - foreach (var s in GetStatements(_classDef)) { - ImportHandler.HandleFromImport(s); - } - foreach (var s in GetStatements(_classDef)) { - ImportHandler.HandleImport(s); - } - UpdateClassMembers(); + if (!Eval.StubOnlyAnalysis) { + // Process imports + foreach (var s in GetStatements(_classDef)) { + ImportHandler.HandleFromImport(s); + } - // Process assignments so we get class variables declared. - // Note that annotated definitions and assignments can be intermixed - // and must be processed in order. Consider - // class A: - // x: int - // x = 1 - foreach (var s in GetStatements(_classDef)) { - switch (s) { - case AssignmentStatement assignment: - AssignmentHandler.HandleAssignment(assignment, LookupOptions.All); - break; - case ExpressionStatement e: - AssignmentHandler.HandleAnnotatedExpression(e.Expression as ExpressionWithAnnotation, null, LookupOptions.All); - break; + foreach (var s in GetStatements(_classDef)) { + ImportHandler.HandleImport(s); } - } - UpdateClassMembers(); - // Ensure constructors are processed so class members are initialized. - EvaluateConstructors(_classDef); - UpdateClassMembers(); + UpdateClassMembers(); + + // Process assignments so we get class variables declared. + // Note that annotated definitions and assignments can be intermixed + // and must be processed in order. Consider + // class A: + // x: int + // x = 1 + foreach (var s in GetStatements(_classDef)) { + switch (s) { + case AssignmentStatement assignment: + AssignmentHandler.HandleAssignment(assignment, LookupOptions.All); + break; + case ExpressionStatement e: + AssignmentHandler.HandleAnnotatedExpression(e.Expression as ExpressionWithAnnotation, null, LookupOptions.All); + break; + } + } + + UpdateClassMembers(); + + // Ensure constructors are processed so class members are initialized. + EvaluateConstructors(_classDef); + UpdateClassMembers(); + } // Process remaining methods. SymbolTable.EvaluateScope(_classDef); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 7d4ebbf9d..393fc3274 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -46,6 +46,10 @@ public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) private FunctionDefinition FunctionDefinition { get; } public override void Evaluate() { + if(Eval.StubOnlyAnalysis) { + return; + } + var stub = _function.DeclaringModule.ModuleType == ModuleType.Stub; using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index c9c99e2e0..8aacf01e1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -61,9 +61,7 @@ public override bool Walk(ClassDefinition cd) { // The variable is transient (non-user declared) hence it does not have location. // Class type is tracking locations for references and renaming. _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration); - if (!_eval.StubOnlyAnalysis) { - _table.Add(new ClassEvaluator(_eval, cd)); - } + _table.Add(new ClassEvaluator(_eval, cd)); // Open class scope _scopes.Push(_eval.OpenScope(_eval.Module, cd, out _)); } From e7a2bd5744b29507298246c3fb864d6d80d22567 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 17 Dec 2019 14:39:04 -0800 Subject: [PATCH 099/102] Test fix --- src/LanguageServer/Test/LinterTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/LanguageServer/Test/LinterTests.cs b/src/LanguageServer/Test/LinterTests.cs index 678b12c46..b996762fb 100644 --- a/src/LanguageServer/Test/LinterTests.cs +++ b/src/LanguageServer/Test/LinterTests.cs @@ -45,6 +45,12 @@ public async Task LinterOnOff() { var d = a.LintModule(analysis.Document); d.Should().HaveCount(1); + + var aop = Services.GetService(); + if (aop != null) { + Services.RemoveService(aop); + } + var provider = Substitute.For(); Services.AddService(provider); From cbac83fb5c8f726117b60a695a775e0996abfd84 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 18 Dec 2019 21:00:59 -0800 Subject: [PATCH 100/102] Fix tests --- .../Definitions/IModuleDatabaseService.cs | 10 ------- src/Caching/Impl/ModuleDatabase.cs | 26 +------------------ src/Caching/Test/RestoreTests.cs | 18 ++++++++++++- 3 files changed, 18 insertions(+), 36 deletions(-) diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index e04965a97..5e2258cf9 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -36,15 +36,5 @@ internal interface IModuleDatabaseService: IModuleDatabaseCache { /// /// Cancellation token Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default); - - /// - /// Determines if module analysis exists in the storage. - /// - bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType); - } - - internal static class ModuleDatabaseExtensions { - public static bool ModuleExistsInStorage(this IModuleDatabaseService dbs, IPythonModule module) - => dbs.ModuleExistsInStorage(module.Name, module.FilePath, module.ModuleType); } } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 2e51aedde..6bfea9572 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -47,7 +47,6 @@ private readonly ConcurrentDictionary _searchResults private readonly IFileSystem _fs; private readonly AnalysisCachingLevel _defaultCachingLevel; private readonly CacheWriter _cacheWriter; - private AnalysisCachingLevel? _cachingLevel; public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { _services = sm; @@ -76,27 +75,6 @@ public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleT ? RestoreModule(model) : null; } - /// - /// Determines if module analysis exists in the storage. - /// - public bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType) { - if (GetCachingLevel() == AnalysisCachingLevel.None) { - return false; - } - - var key = new AnalysisModuleKey(name, filePath); - if (_searchResults.TryGetValue(key, out var result)) { - return result; - } - - WithRetries.Execute(() => { - var dbPath = FindDatabaseFile(name, filePath, moduleType); - _searchResults[key] = result = !string.IsNullOrEmpty(dbPath); - return result; - }, "Unable to find database file for {name}.", _log); - return false; - } - public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default) { var cachingLevel = GetCachingLevel(); if (cachingLevel == AnalysisCachingLevel.None) { @@ -192,9 +170,7 @@ private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel } private AnalysisCachingLevel GetCachingLevel() - => _cachingLevel - ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) - ?? _defaultCachingLevel; + => _services.GetService()?.Options.AnalysisCachingLevel ?? _defaultCachingLevel; public void Dispose() => _cacheWriter.Dispose(); } diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index b7883c0c4..1bb746f57 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -22,6 +22,7 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; using TestUtilities; namespace Microsoft.Python.Analysis.Caching.Tests { @@ -57,6 +58,8 @@ def func2() -> C2: ... var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); Services.AddService(dbs); + + EnableCaching(); await dbs.StoreModuleAnalysisAsync(analysis2, immediate: true, CancellationToken.None); await Services.GetService().ResetAnalyzer(); @@ -101,15 +104,28 @@ private async Task CreateDatabaseAsync(IPythonInterpreter interpreter) { var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); Services.AddService(dbs); + EnableCaching(); + var importedModules = interpreter.ModuleResolution.GetImportedModules(); foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { await dbs.StoreModuleAnalysisAsync(m.Analysis, immediate: true); } - + importedModules = interpreter.TypeshedResolution.GetImportedModules(); foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { await dbs.StoreModuleAnalysisAsync(m.Analysis, immediate: true); } } + + private void EnableCaching() { + var a = Services.GetService(); + Services.RemoveService(a); + var aop = Substitute.For(); + aop.Options.Returns(x => new AnalysisOptions { + AnalysisCachingLevel = AnalysisCachingLevel.Library, + StubOnlyAnalysis = a.Options.StubOnlyAnalysis + }); + Services.AddService(aop); + } } } From 85c5b0cf999ee509f50463dde8e5746dd36399ea Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 2 Jan 2020 09:53:43 -0800 Subject: [PATCH 101/102] Test update --- src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs | 6 +++++- src/Analysis/Ast/Test/AnalysisTestBase.cs | 2 +- src/Caching/Test/AnalysisCachingTestBase.cs | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index e1ae3ae88..7613412fd 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -70,7 +70,11 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public IPythonType UnknownType => Interpreter.UnknownType; public Location DefaultLocation { get; } public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule; - public bool StubOnlyAnalysis => _analysisOptions.StubOnlyAnalysis && Module.Stub != null && Module.ModuleType != ModuleType.User; + + public bool StubOnlyAnalysis + => _analysisOptions.StubOnlyAnalysis + && Module.Stub != null && Module.ModuleType != ModuleType.User && Module.Stub.IsTypeshed; + public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 28a29e6bf..a2d9bf6a7 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -96,7 +96,7 @@ protected async Task CreateServicesAsync(string root, Interpret var ap = Substitute.For(); ap.Options.Returns(x => new AnalysisOptions { AnalysisCachingLevel = AnalysisCachingLevel.None, - StubOnlyAnalysis = true, + StubOnlyAnalysis = false, LintingEnabled = true }); sm.AddService(ap); diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 9ebca86f6..a8faa5dc0 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -20,7 +20,6 @@ using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Tests; using Microsoft.Python.Analysis.Types; using Newtonsoft.Json; From e68117fbf0a37cbfeb12e9686c8d510675f84915 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 5 May 2020 08:17:06 -0700 Subject: [PATCH 102/102] Fix merge --- src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 4dfa6d3f0..be2bfdd9d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -56,7 +56,6 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs AnalysisCachingLevel = AnalysisCachingLevel.None, StubOnlyAnalysis = true, KeepLibraryAst = false, - KeepLibraryLocalVariables = false, LintingEnabled = true }; }