commit 979e222e68e0de56dcd093e7769a8f9ebda0c804 Author: j Date: Wed Oct 23 17:52:59 2024 +0800 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..73e7f40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +.DS_Store + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8436a38 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +## [SQLFlow](https://sqlflow.gudusoft.com) - A tool that tracks column-level data lineage + +Track Column-Level Data Lineage for [more than 20 major databases](/databases/readme.md) including +Snowflake, Hive, SparkSQL, Teradata, Oracle, SQL Server, AWS redshift, BigQuery, etc. + +Build and visualize lineage from SQL script from query history, ETL script, +Github/Bitbucket, Local filesystem and remote databases. + +[Exploring lineage using an interactive diagram](https://sqlflow.gudusoft.com) or programmatically using [Restful APIs](/api) or [SDKs](https://www.gudusoft.com/sqlflow-java-library-2/). + +Discover data lineage in this query: +```sql +insert into emp (id,first_name,last_name,city,postal_code,ph) + select a.id,a.first_name,a.last_name,a.city,a.postal_code,b.ph + from emp_addr a + inner join emp_ph b on a.id = b.id; +``` + +SQLFlow presents a nice clean graph to you that tells +where the data came from, what transformations it underwent along the way, +and what other data items are derived from this data value. + +[![SQLFlow Introduce](images/sqlflow_introduce1.png)](https://sqlflow.gudusoft.com) + +### What SQLFlow can do for you +- Scan your database and discover the data lineage instantly. +- Automatically collect SQL script from github/bitbucket or local file system. +- Provide a nice cleam diagram to the end-user to understand the data lineage quickly. +- programmatically using [Restful APIs](/api) or [SDKs](https://www.gudusoft.com/sqlflow-java-library-2/) to get lineage in CSV, JSON, Graphml format. +- Incorporate the lineage metadata decoded from the complex SQL script into your own metadata database for further processing. +- Visualize the metadata already existing in your database to release the power of data. +- Perform impact analysis and root-cause analysis by tracing lineage backwards or forwards with several mouse click. +- Able to process SQL script from more than 20 major database vendors. + +### How to use SQLFlow +- Open [the official website](https://gudusoft.com/sqlflow/#/) of the SQLFlow and paste your SQL script or metadata to get a nice clean lineage diagram. +- Call the [Restful API](/api) of the SQLFlow in your own code to get data lineage metadata decoded by the SQLFlow from the SQL script. +- The [on-premise version](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow.md) of SQLflow enables you to use it on your own server to keep the data safer. + + +### Restful APIs +- [SQLFlow API document](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md) +- [Client in C#](https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp) + +### SQLFlow architecture +- [Architecture document](sqlflow_architecture.md) + +### User manual and FAQ +- [User guide](sqlflow_guide.md) +- [SQLFlow FAQ](sqlflow_faq.md) + + diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..92c2695 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,50 @@ +## 一、SQLFlow 是什么 + +数据库中视图(View)的数据来自表(Table)或其他视图，视图中字段(Column)的数据可能来自多个表中多个字段的聚集(aggregation)。 +表中的数据可能通过ETL从外部系统中导入。这种从数据的源头经过各个处理环节，到达数据终点的数据链路关系称为数据血缘关系([data lineage](https://en.wikipedia.org/wiki/Data_lineage))。 + +[SQLFlow](https://sqlflow.gudusoft.com/) 通过分析各种数据库对象的定义(DDL)、DML 语句、ETL/ELT中使用的存储过程(Proceudre,Function)、 +触发器(Trigger)和其他 SQL 脚本，给出完整的数据血缘关系。 + + +在大型数据仓库中，完整的数据血缘关系可以用来进行数据溯源、表和字段变更的影响分析、数据合规性的证明、数据质量的检查等。 + +举例来说，可能会问财务报表中的统计结果，它是有哪些子系统（采购、生产、销售等）提供的数据汇总而成的？ +当某个子系统（例如 销售子系统）的表和字段等数据结构发生变化时，可能会影响其它子系统吗？ +财务报表子系统中的表和字段是否也需要进行相应的改动？ + +SQLFlow 会帮助你回答这些问题，以可视化的图形方式把这些关系呈现在你面前，让你对组织的IT系统中的数据流动一目了然。 + +![SQLFlow Introduce](images/sqlflow_introduce1.png) + +## 二、SQLFlow 是怎样工作的 + +1. 从数据库、版本控制系统、文件系统中获取 SQL 脚本。 +2. 解析 SQL 脚本，分析其中的各种数据库对象关系，建立数据血缘关系。 +3. 以各种形式呈现数据血缘关系，包括交互式 UI、CSV、JSON、GRAPHML 格式。 + +## 三、SQLFlow 的组成 + +1. Backend， 后台由一系列 Java 程序组成。负责 SQL 的解析、数据血缘分析、可视化元素的布局、身份认证等。 +2. Frontend，前端由一系列 javascript、html 代码组成。负责 SQL 的递交、数据血缘关系的可视化展示。 +3. [Grabit 工具](https://www.gudusoft.com/grabit/)，一个 Java 程序。负责从数据库、版本控制系统、文件系统中收集 SQL 脚本，递交给后台进行数据血缘分析。 +4. [Restful API](https://github.com/sqlparser/sqlflow_public/tree/master/api)，一套完整的 API。让用户可以通过 Java、C#、Python、PHP 等编程语言与后台进行交互，完成数据血缘分析。 + +![SQLFlow Components](https://github.com/sqlparser/sqlflow_public/raw/master/sqlflow_components.png) + +## 四、SQLFlow的使用 + +1. 通过浏览器访问[SQLFlow的前端](https://sqlflow.gudusoft.com/)。 +2. 在浏览器中上传SQL文本或文件。 +3. 点击分析按钮后，查看数据血缘关系的可视化结果。 +4. 在浏览器中，以交互形式，查看特定表或视图的完整血缘关系图。 +5. 用 grabit 工具或 API，提交需要处理的 SQL 文件，然后在浏览器中查看结果，或在自己的代码中对返回的结果做进一步处理。 + +## 五、SQLFlow 的局限 + +SQLFlow 仅仅通过分析 SQL 脚本，包含存储过程(proceudre, function, trigger)来获取数据库中的数据血缘关系。 +但在 ETL 数据转换过程中，会用到很多其它技术和工具，由此产生的数据血缘关系目前 SQLFlow 无法探知。 + +## 六、进一步了解 SQLFlow +1. 支持多达21个主流数据库 +2. [Architecture document](sqlflow_architecture.md) \ No newline at end of file diff --git a/api/csharp/.gitignore b/api/csharp/.gitignore new file mode 100644 index 0000000..c9794b9 --- /dev/null +++ b/api/csharp/.gitignore @@ -0,0 +1,344 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj +!linux.pubxml +!osx.pubxml +!win.pubxml + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ +# ASP.NET Core default setup: bower directory is configured as wwwroot/lib/ and bower restore is true +**/wwwroot/lib/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb diff --git a/api/csharp/SQLFlowClient.sln b/api/csharp/SQLFlowClient.sln new file mode 100644 index 0000000..edb5005 --- /dev/null +++ b/api/csharp/SQLFlowClient.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29503.13 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SQLFlowClient", "SQLFlowClient\SQLFlowClient.csproj", "{8F80B6E9-F33B-4936-8111-48A9BCA9AEDC}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {8F80B6E9-F33B-4936-8111-48A9BCA9AEDC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8F80B6E9-F33B-4936-8111-48A9BCA9AEDC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8F80B6E9-F33B-4936-8111-48A9BCA9AEDC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8F80B6E9-F33B-4936-8111-48A9BCA9AEDC}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {3C9B0DCD-4A60-4E0C-9A35-7211C074B0D1} + EndGlobalSection +EndGlobal diff --git a/api/csharp/SQLFlowClient/.gitignore b/api/csharp/SQLFlowClient/.gitignore new file mode 100644 index 0000000..53c37a1 --- /dev/null +++ b/api/csharp/SQLFlowClient/.gitignore @@ -0,0 +1 @@ +dist \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/Config.cs b/api/csharp/SQLFlowClient/Config.cs new file mode 100644 index 0000000..283061c --- /dev/null +++ b/api/csharp/SQLFlowClient/Config.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SQLFlowClient +{ + class Config + { + public string Host { get; set; } + public string Token { get; set; } + public string SecretKey { get; set; } + public string UserId { get; set; } + } +} diff --git a/api/csharp/SQLFlowClient/DBVendor.cs b/api/csharp/SQLFlowClient/DBVendor.cs new file mode 100644 index 0000000..c2325ba --- /dev/null +++ b/api/csharp/SQLFlowClient/DBVendor.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.ComponentModel; + +namespace SQLFlowClient +{ + public enum DBVendor + { + bigquery, + couchbase, + db2, + greenplum, + hana , + hive, + impala , + informix, + mdx, + mysql, + netezza, + openedge, + oracle, + postgresql, + redshift, + snowflake, + mssql, + sybase, + teradata, + vertica, + } +} diff --git a/api/csharp/SQLFlowClient/HttpService.cs b/api/csharp/SQLFlowClient/HttpService.cs new file mode 100644 index 0000000..302fadf --- /dev/null +++ b/api/csharp/SQLFlowClient/HttpService.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using System.Linq; +using System.Net.Http.Headers; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.IO; +using System.Diagnostics; + +namespace SQLFlowClient +{ + public static class HttpService + { + private static Config config; + + public static async Task Request(Options options) + { + config = new Config + { + Host = "https://api.gudusoft.com", + Token = "", + UserId = "gudu|0123456789", + }; + try + { + if (File.Exists("./config.json")) + { + var json = JObject.Parse(File.ReadAllText("./config.json")); + if (!string.IsNullOrWhiteSpace(json["Host"]?.ToString())) + { + config.Host = json["Host"].ToString(); + } + if (!string.IsNullOrWhiteSpace(json["Token"]?.ToString())) + { + config.Token = json["Token"].ToString(); + } + if (!string.IsNullOrWhiteSpace(json["SecretKey"]?.ToString())) + { + config.SecretKey = json["SecretKey"].ToString(); + } + if (!string.IsNullOrWhiteSpace(json["UserId"]?.ToString())) + { + config.UserId = json["UserId"].ToString(); + } + } + } + catch (Exception e) + { + Console.WriteLine($"Invalid config.json :\n{e.Message}"); + return; + } + //if (!string.IsNullOrWhiteSpace(options.Token)) + //{ + // config.Token = options.Token; + //} + //if (!string.IsNullOrWhiteSpace(options.UserId)) + //{ + // config.UserId = options.UserId; + //} + //if (!string.IsNullOrWhiteSpace(options.SecretKey)) + //{ + // config.SecretKey = options.SecretKey; + //} + if (options.Version) + { + await Version(); + } + else + { + await SQLFlow(options); + } + } + + public static async Task SQLFlow(Options options) + { + StreamContent sqlfile; + if (options.SQLFile == null) + { + Console.WriteLine($"Please specify an input file. (e.g. SQLFlowClient test.sql)"); + return; + } + try + { + string path = Path.GetFullPath(options.SQLFile); + sqlfile = new StreamContent(File.Open(options.SQLFile, FileMode.Open)); + } + catch (Exception e) + { + Console.WriteLine($"Open file failed.\n{e.Message}"); + return; + } + var types = options.ShowRelationType.Split(",") + .Where(p => Enum.GetNames(typeof(RelationType)).FirstOrDefault(t => t.ToLower() == p.ToLower()) == null) + .ToList(); + if (types.Count != 0) + { + Console.WriteLine($"Wrong relation type : { string.Join(",", types) }.\nIt should be one or more from the following list : fdd, fdr, frd, fddi, join"); + return; + } + string dbvendor = Enum.GetNames(typeof(DBVendor)).FirstOrDefault(p => p.ToLower() == options.DBVendor.ToLower()); + if (dbvendor == null) + { + Console.WriteLine($"Wrong database vendor : {options.DBVendor}.\nIt should be one of the following list : " + + $"bigquery, couchbase, db2, greenplum, hana , hive, impala , informix, mdx, mysql, netezza, openedge," + + $" oracle, postgresql, redshift, snowflake, mssql, sybase, teradata, vertica"); + return; + } + if (!string.IsNullOrWhiteSpace(config.SecretKey) && !string.IsNullOrWhiteSpace(config.UserId)) + { + // request token + string url2 = $"{config.Host}/gspLive_backend/user/generateToken"; + using var client2 = new HttpClient(); + using var response2 = await client2.PostAsync(url2, content: new FormUrlEncodedContent(new List> + { + new KeyValuePair("userId", config.UserId), + new KeyValuePair("secretKey", config.SecretKey) + })); + if (response2.IsSuccessStatusCode) + { + var text = await response2.Content.ReadAsStringAsync(); + var jobject = JObject.Parse(text); + var json = jobject.ToString(); + var code = jobject.SelectToken("code"); + if (code?.ToString() == "200") + { + config.Token = jobject.SelectToken("token").ToString(); + } + else + { + Console.WriteLine($"{url2} error, code={code?.ToString() }"); + return; + } + } + else + { + Console.WriteLine($"Wrong response code {(int)response2.StatusCode} {response2.StatusCode}.url={url2}"); + return; + } + + } + var form = new MultipartFormDataContent{ + { sqlfile , "sqlfile" , "sqlfile" }, + { new StringContent("dbv"+dbvendor) , "dbvendor" }, + { new StringContent(options.ShowRelationType) , "showRelationType" }, + { new StringContent(options.SimpleOutput.ToString()) , "simpleOutput" }, + { new StringContent(options.IgnoreRecordSet.ToString()) , "ignoreRecordSet" }, + { new StringContent(options.ignoreFunction.ToString()) , "ignoreFunction" }, + { new StringContent(config.UserId) , "userId" }, + { new StringContent(config.Token) , "token" }, + }; + try + { + var stopWatch = Stopwatch.StartNew(); + string url = $"{config.Host}/gspLive_backend/sqlflow/generation/sqlflow/" + (options.IsGraph ? "graph" : ""); + using var client = new HttpClient(); + // client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", config.Token); + using var response = await client.PostAsync(url, form); + if (response.IsSuccessStatusCode) + { + stopWatch.Stop(); + var text = await response.Content.ReadAsStringAsync(); + var result = new SQLFlowResult(text); + if (result.data && result.dbobjs || result.data && result.sqlflow && result.graph) + { + if (options.Output != "") + { + try + { + File.WriteAllText(Path.GetFullPath(options.Output), result.json); + Console.WriteLine($"Output has been saved to {options.Output}."); + } + catch (Exception e) + { + Console.WriteLine($"Save File failed.{e.Message}"); + } + } + else + { + Console.WriteLine(result.json ?? ""); + } + } + if (result.error) + { + Console.WriteLine($"Success with some errors.Executed in {stopWatch.Elapsed.TotalSeconds.ToString("0.00")} seconds by host {config.Host}."); + } + else + { + Console.WriteLine($"Success.Executed in {stopWatch.Elapsed.TotalSeconds.ToString("0.00")} seconds by host {config.Host}."); + } + } + else + { + Console.WriteLine($"Wrong response code {(int)response.StatusCode} {response.StatusCode}."); + } + } + catch (Exception e) + { + Console.WriteLine($"An unknonwn exeception occurs :\n{e.Message}"); + } + } + + public static async Task Version() + { + try + { + string url = $"{config.Host}/gspLive_backend/version"; + using var client = new HttpClient(); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", config.Token); + var form = new MultipartFormDataContent{ + { new StringContent(config.UserId) , "userId" }, + }; + using var response = await client.PostAsync(url, form); + if (response.IsSuccessStatusCode) + { + var text = await response.Content.ReadAsStringAsync(); + var json = JObject.Parse(text); + var gsp = new + { + ReleaseDate = json.SelectToken("version.gsp.['release.date']")?.ToString(), + version = json.SelectToken("version.gsp.version")?.ToString(), + }; + var backend = new + { + ReleaseDate = json.SelectToken("version.backend.['release.date']")?.ToString(), + version = json.SelectToken("version.backend.version")?.ToString(), + }; + Console.WriteLine(" version relase date"); + Console.WriteLine("SQLFlowClient 1.2.0 2020/12/13"); + Console.WriteLine($"gsp {gsp.version} {gsp.ReleaseDate}"); + Console.WriteLine($"backend {backend.version} {backend.ReleaseDate}"); + } + else + { + Console.WriteLine($"Not connected.Wrong response code {(int)response.StatusCode} {response.StatusCode}."); + } + } + catch (Exception e) + { + Console.WriteLine($"An unknonwn exeception occurs :\n{e.Message}"); + } + } + } +} diff --git a/api/csharp/SQLFlowClient/Program.cs b/api/csharp/SQLFlowClient/Program.cs new file mode 100644 index 0000000..aa46c94 --- /dev/null +++ b/api/csharp/SQLFlowClient/Program.cs @@ -0,0 +1,73 @@ +using System; +using CommandLine; +using CommandLine.Text; + +namespace SQLFlowClient +{ + public class Options + { + [Value(0, MetaName = "sqlfile", Required = false, HelpText = "Input sqlfile to be processed.")] + public string SQLFile { get; set; } + + [Option('g', "graph", Required = false, Default = false, HelpText = "Get the graph from sql.")] + public bool IsGraph { get; set; } + + [Option('v', "dbvendor", Required = false, Default = "oracle", HelpText = "Set the database of the sqlfile.")] + public string DBVendor { get; set; } + + [Option('r', "showRelationType", Required = false, Default = "fdd", HelpText = "Set the relation type.")] + public string ShowRelationType { get; set; } + + [Option('s', "simpleOutput", Required = false, Default = false, HelpText = "Set whether to get simple output.")] + public bool SimpleOutput { get; set; } + + [Option("ignoreRecordSet", Required = false, Default = false, HelpText = "Set whether to ignore record set.")] + public bool IgnoreRecordSet { get; set; } + + [Option("ignoreFunction", Required = false, Default = false, HelpText = "Set whether to ignore function.")] + public bool ignoreFunction { get; set; } + + [Option('o', "output", Required = false, Default = "", HelpText = "Save output as a file.")] + public string Output { get; set; } + + //[Option('t', "token", Required = false, Default = "", HelpText = "If userId and secretKey is given, token will be ignore, otherwise it will use token.")] + //public string Token { get; set; } + + //[Option('u', "userId", Required = false, Default = "", HelpText = "")] + //public string UserId { get; set; } + + //[Option('k', "secretKey", Required = false, Default = "", HelpText = "")] + //public string SecretKey { get; set; } + + [Option("version", Required = false, Default = false, HelpText = "Show version.")] + public bool Version { get; set; } + } + + class Program + { + static void Main(string[] args) + { + var parser = new Parser(with => + { + with.AutoVersion = false; + with.AutoHelp = true; + }); + var parserResult = parser.ParseArguments(args); ; + parserResult + .WithParsed(options => + { + HttpService.Request(options).Wait(); + }) + .WithNotParsed(errs => + { + var helpText = HelpText.AutoBuild(parserResult, h => + { + h.AutoHelp = true; + h.AutoVersion = false; + return h; + }, e => e); + Console.WriteLine(helpText); + }); + } + } +} diff --git a/api/csharp/SQLFlowClient/Properties/PublishProfiles/linux.pubxml b/api/csharp/SQLFlowClient/Properties/PublishProfiles/linux.pubxml new file mode 100644 index 0000000..b88256f --- /dev/null +++ b/api/csharp/SQLFlowClient/Properties/PublishProfiles/linux.pubxml @@ -0,0 +1,17 @@ + + + + + Release + Any CPU + bin\Release\netcoreapp3.0\publish\linux + FileSystem + netcoreapp3.0 + linux-x64 + true + True + False + + \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/Properties/PublishProfiles/osx.pubxml b/api/csharp/SQLFlowClient/Properties/PublishProfiles/osx.pubxml new file mode 100644 index 0000000..6f66028 --- /dev/null +++ b/api/csharp/SQLFlowClient/Properties/PublishProfiles/osx.pubxml @@ -0,0 +1,17 @@ + + + + + Release + Any CPU + bin\Release\netcoreapp3.0\publish\osx\ + FileSystem + netcoreapp3.0 + osx-x64 + true + True + False + + \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/Properties/PublishProfiles/win.pubxml b/api/csharp/SQLFlowClient/Properties/PublishProfiles/win.pubxml new file mode 100644 index 0000000..61d064c --- /dev/null +++ b/api/csharp/SQLFlowClient/Properties/PublishProfiles/win.pubxml @@ -0,0 +1,18 @@ + + + + + Release + Any CPU + bin\Release\netcoreapp3.0\publish\win\ + FileSystem + netcoreapp3.0 + win-x64 + true + True + False + False + + \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/Properties/launchSettings.json b/api/csharp/SQLFlowClient/Properties/launchSettings.json new file mode 100644 index 0000000..b8f6f0f --- /dev/null +++ b/api/csharp/SQLFlowClient/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "SQLFlowClient": { + "commandName": "Project", + "commandLineArgs": "test.sql" + } + } +} \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/RelationType.cs b/api/csharp/SQLFlowClient/RelationType.cs new file mode 100644 index 0000000..34ee836 --- /dev/null +++ b/api/csharp/SQLFlowClient/RelationType.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SQLFlowClient +{ + public enum RelationType + { + fdd, + fdr, + frd, + fddi, + join, + } +} diff --git a/api/csharp/SQLFlowClient/SQLFlowClient.csproj b/api/csharp/SQLFlowClient/SQLFlowClient.csproj new file mode 100644 index 0000000..e050c66 --- /dev/null +++ b/api/csharp/SQLFlowClient/SQLFlowClient.csproj @@ -0,0 +1,30 @@ + + + + Exe + netcoreapp3.0 + 1.0.9 + 1.0.9.0 + 1.0.9.0 + + + + + + + + + + + + + + + + Always + + + Always + + + diff --git a/api/csharp/SQLFlowClient/SQLFlowResult.cs b/api/csharp/SQLFlowClient/SQLFlowResult.cs new file mode 100644 index 0000000..e1f9883 --- /dev/null +++ b/api/csharp/SQLFlowClient/SQLFlowResult.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace SQLFlowClient +{ + class SQLFlowResult + { + private readonly int maxLength = 24814437;// json will not be formatted if the string length exceeds this number + public string json; + public bool data; + public bool error; + public bool dbobjs; + public bool sqlflow; + public bool graph; + + public SQLFlowResult(string text) + { + if (text.Length <= maxLength) + { + var jobject = JObject.Parse(text); + json = jobject.ToString(); + data = jobject.SelectToken("data") != null; + error = jobject.SelectToken("error") != null; + dbobjs = jobject.SelectToken("data.dbobjs") != null; + sqlflow = jobject.SelectToken("data.sqlflow") != null; + graph = jobject.SelectToken("data.graph") != null; + } + else + { + json = text; + data = false; + error = false; + dbobjs = false; + sqlflow = false; + graph = false; + + using var reader = new JsonTextReader(new StringReader(text)); + while (reader.Read()) + { + if (reader.Value != null) + { + //Console.WriteLine("Token: {0}, Value: {1} ,Depth：{2}", reader.TokenType, reader.Value, reader.Depth); + if (reader.Depth > 3) + { + goto End; + } + if (reader.TokenType.ToString() == "PropertyName") + { + switch (reader.Value.ToString()) + { + case "data": + data = true; + break; + case "error": + error = true; + break; + case "dbobjs": + dbobjs = true; + break; + case "sqlflow": + sqlflow = true; + break; + case "graph": + graph = true; + break; + } + } + } + else + { + //Console.WriteLine("Token: {0}", reader.TokenType); + if (error || dbobjs || sqlflow || graph) + { + reader.Skip(); + } + } + } + End: { } + } + } + } +} diff --git a/api/csharp/SQLFlowClient/config.json b/api/csharp/SQLFlowClient/config.json new file mode 100644 index 0000000..1b5c3aa --- /dev/null +++ b/api/csharp/SQLFlowClient/config.json @@ -0,0 +1,5 @@ +{ + "Host": "https://api.gudusoft.com", + "SecretKey": "d126d0fb1a5a13abb97b160d571f29a2bbaa13861219082da7e9c4d62553ed7c", + "UserId": "auth0|600acd55e68a290069f8a8db" +} \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/publish.bat b/api/csharp/SQLFlowClient/publish.bat new file mode 100644 index 0000000..4c182ca --- /dev/null +++ b/api/csharp/SQLFlowClient/publish.bat @@ -0,0 +1,5 @@ +dotnet publish -c Release /p:PublishProfile=Properties\PublishProfiles\linux.pubxml +dotnet publish -c Release /p:PublishProfile=Properties\PublishProfiles\osx.pubxml +dotnet publish -c Release /p:PublishProfile=Properties\PublishProfiles\win.pubxml +if exist dist rd dist /S /Q +xcopy /s .\bin\Release\netcoreapp3.0\publish .\dist\ \ No newline at end of file diff --git a/api/csharp/SQLFlowClient/test.sql b/api/csharp/SQLFlowClient/test.sql new file mode 100644 index 0000000..b626de2 --- /dev/null +++ b/api/csharp/SQLFlowClient/test.sql @@ -0,0 +1,33 @@ +CREATE VIEW vsal +AS + SELECT a.deptno "Department", + a.num_emp / b.total_count "Employees", + a.sal_sum / b.total_sal "Salary" + FROM (SELECT deptno, + Count() num_emp, + SUM(sal) sal_sum + FROM scott.emp + WHERE city = 'NYC' + GROUP BY deptno) a, + (SELECT Count() total_count, + SUM(sal) total_sal + FROM scott.emp + WHERE city = 'NYC') b +; + +INSERT ALL + WHEN ottl < 100000 THEN + INTO small_orders + VALUES(oid, ottl, sid, cid) + WHEN ottl > 100000 and ottl < 200000 THEN + INTO medium_orders + VALUES(oid, ottl, sid, cid) + WHEN ottl > 200000 THEN + into large_orders + VALUES(oid, ottl, sid, cid) + WHEN ottl > 290000 THEN + INTO special_orders +SELECT o.order_id oid, o.customer_id cid, o.order_total ottl, +o.sales_rep_id sid, c.credit_limit cl, c.cust_email cem +FROM orders o, customers c +WHERE o.customer_id = c.customer_id; \ No newline at end of file diff --git a/api/csharp/readme.md b/api/csharp/readme.md new file mode 100644 index 0000000..e5a30a5 --- /dev/null +++ b/api/csharp/readme.md @@ -0,0 +1,115 @@ +# Get Started +### [Download](https://sqlflow.gudusoft.com/download/) the executable program according to your operating system. + +- [windows](https://sqlflow.gudusoft.com/download/win/SQLFlowClient.exe) +- [mac](https://sqlflow.gudusoft.com/download/osx/SQLFlowClient) +- [linux](https://sqlflow.gudusoft.com/download/linux/SQLFlowClient) + + +### Configuration + +#### SQLFlow Cloud server + +Create a file named `config.json` in directory where the executable(.exe) exists, and then input your `SecretKey` and `UserId`, always set `host` to `https://api.gudusoft.com` ,for example: + +```json +{ + "Host": "https://api.gudusoft.com", + "SecretKey": "XXX", + "UserId": "XXX" +} +``` +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + +#### SQLFlow on-premise version + +Create a file named `config.json` in directory where the executable(.exe) exists, and always set `userId` to `gudu|0123456789`, keep `userSecret` empty, and set `host`to your server ip, for example: + +```json +{ + "Host": "http://your server ip:8081", + "SecretKey": "", + "UserId": "gudu|0123456789" +} +``` +Please [check here](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow.md) to see how to install SQLFlow on-premise version on you own server. + +### Set permissions + + +For mac: +``` +chmod +x SQLFlowClient +``` + +For linux: +``` +chmod +x SQLFlowClient +``` + +### Create a simple sql file for testing +For example, test.sql: +```sql +insert into t2 select * from t1; +``` + +Run the program from command line: +``` +./SQLFlowClient test.sql +``` +``` +./SQLFlowClient test.sql -g +``` + +# Usage + +SQLFlowClient filepath -parameter value + +### parameters + +| parameter | short | value type | default | | +| ------------------ | ----- | ------------------------------------------------------------ | ------- | --------------------------------- | +| --graph | -g | boolean | false | Get the graph from sql. | +| --dbvendor | -v | one of the following list :
bigquery, couchbase, db2, greenplum,
hana , hive, impala , informix,
mdx, mysql, netezza, openedge,
oracle, postgresql, redshift, snowflake,
mssql, sybase, teradata, vertica | oracle | Set the database of the sqlfile. | +| --showRelationType | -r | one or more from the following list :
fdd, fdr, frd, fddi, join | fdd | Set the relation type. | +| --simpleOutput | -s | boolean | false | Set whether to get simple output. | +| --ignoreRecordSet | | boolean | false | Set whether to ignore record set. | +| --ignoreFunction | | boolean | false | Set whether to ignore function. | +| --output | -o | string | "" | Save output as a file. | +| --help | | | | Display this help screen. | +| --version | | | | Display version information. | + +### examples +1. SQLFlowClient test.sql +2. SQLFlowClient test.sql -g +3. SQLFlowClient test.sql -g -v oracle +4. SQLFlowClient test.sql -g -v oracle -r fdr +5. SQLFlowClient test.sql -g -v oracle -r fdr,join +6. SQLFlowClient test.sql -g -v oracle -r fdr,join -s +7. SQLFlowClient test.sql -g -v oracle -r fdr,join -s --ignoreRecordSet +8. SQLFlowClient test.sql -g -v oracle -r fdr,join -s --ignoreFunction -o result.txt + +# Compile and build on windows + +### Download and install the .NET Core SDK + +``` +https://dotnet.microsoft.com/download +``` + +### Download source code +``` +git clone https://github.com/sqlparser/sqlflow_public.git +``` + +### Build from command line + +``` +dotnet publish -c Release /p:PublishProfile=Properties\PublishProfiles\linux.pubxml +dotnet publish -c Release /p:PublishProfile=Properties\PublishProfiles\osx.pubxml +dotnet publish -c Release /p:PublishProfile=Properties\PublishProfiles\win.pubxml +``` + +### [Download executable programs](https://sqlflow.gudusoft.com/download//) + diff --git a/api/firstdemo.py b/api/firstdemo.py new file mode 100644 index 0000000..c5c79e4 --- /dev/null +++ b/api/firstdemo.py @@ -0,0 +1,88 @@ +""" +How to get user_id and secret_key: https://docs.gudusoft.com/3.-api-docs/prerequisites#generate-account-secret + +once you have user_id and secret_key, + +user_id: +secret_key: + +you can get token by: + +curl -X POST "https://api.gudusoft.com/gspLive_backend/user/generateToken" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:application/x-www-form-urlencoded;charset=UTF-8" -d "secretKey=YOUR SECRET KEY" -d "userId=YOUR USER ID HERE" + +and then you can use the token to call the api: +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow?showRelationType=fdd" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "sqlfile=" -F "dbvendor=dbvoracle" -F "ignoreRecordSet=false" -F "simpleOutput=false" -F "sqltext=CREATE VIEW vsal as select * from emp" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" + +""" + +# Python code to call the API based on the description: + +import requests +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +# Function to get the token +def get_token(user_id, secret_key): + url = "https://api.gudusoft.com/gspLive_backend/user/generateToken" + headers = { + "Request-Origion": "testClientDemo", + "accept": "application/json;charset=utf-8", + "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8" + } + data = { + "secretKey": secret_key, + "userId": user_id + } + response = requests.post(url, headers=headers, data=data, verify=False, proxies=None) + + # Check if the request was successful + response.raise_for_status() + + # Parse the JSON response + json_response = response.json() + + # Check if 'token' key exists directly in the response + if 'token' in json_response: + return json_response['token'] + else: + raise ValueError("Token not found in the response.") + +# Function to call the SQLFlow API +def call_sqlflow_api(user_id, token, sql_text): + url = "https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow?showRelationType=fdd" + headers = { + "Request-Origion": "testClientDemo", + "accept": "application/json;charset=utf-8" + } + data = { + "sqlfile": "", + "dbvendor": "dbvoracle", + "ignoreRecordSet": "false", + "simpleOutput": "false", + "sqltext": sql_text, + "userId": user_id, + "token": token + } + response = requests.post(url, headers=headers, data=data) + return response.json() + +# Example usage +# How to get user_id and secret_key: https://docs.gudusoft.com/3.-api-docs/prerequisites#generate-account-secret + +user_id = "your user id" +secret_key = "your secret key" +sql_text = "CREATE VIEW vsal AS SELECT * FROM emp" + +try: + # Get the token + token = get_token(user_id, secret_key) + print("Token:", token) +except requests.exceptions.RequestException as e: + print("Error making request:", e) +except ValueError as e: + print("Error parsing response:", e) + +# Call the SQLFlow API +result = call_sqlflow_api(user_id, token, sql_text) +print(result) \ No newline at end of file diff --git a/api/java/DataLineageParser.java b/api/java/DataLineageParser.java new file mode 100644 index 0000000..90172e4 --- /dev/null +++ b/api/java/DataLineageParser.java @@ -0,0 +1,90 @@ +package java; +/** +* 解析SQLFLow exportLineageAsJson接口返回的JSON格式的血缘关系中的关系链路 +* +* 例如demo中的血缘数据，解析成以下链路： +* 达成的目标是，List中两个元素： +* SCOTT.DEPT -> SCOTT.EMP->VSAL +* SCOTT.EMP->VSAL +*/ + +public class DataLineageParser { + static class Node { + String value; + String id; + Node next; + + public Node(String value, String id) { + this.value = value; + this.id = id; + } + + public String key() { + Node node = this.next; + StringBuilder key = new StringBuilder(id); + while (node != null) { + key.append(node.id); + node = node.next; + } + return key.toString(); + } + } + + public static void main(String[] args) { + String input = "{"jobId":"d9550e491c024d0cbe6e1034604aca17","code":200,"data":{"mode":"global","sqlflow":{"relationship":[{"sources":[{"parentName":"ORDERS","column":"TABLE","coordinates":[],"id":"10000106","parentId":"86"}],"id":"1000012311","type":"fdd","target":{"parentName":"SPECIAL_ORDERS","column":"TABLE","coordinates":[],"id":"10000102","parentId":"82"}},{"sources":[{"parentName":"CUSTOMERS","column":"TABLE","coordinates":[],"id":"10000103","parentId":"94"}],"id":"1000012312","type":"fdd","target":{"parentName":"SPECIAL_ORDERS","column":"TABLE","coordinates":[],"id":"10000102","parentId":"82"}}]}},"sessionId":"8bb7d3da4b687bb7badf01608a739fbebd61309cd5a643cecf079d122095738a_1685604216451"}"; + try { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(input); + JsonNode relationshipNode = jsonNode.path("data").path("sqlflow").path("relationships"); + List> dataList = objectMapper.readValue(relationshipNode.toString(), new TypeReference>>() { + }); + + ArrayList value = new ArrayList<>(); + Map nodeMap = new HashMap<>(); + for (Map data : dataList) { + List> sources = (List>) data.get("sources"); + Map targetNode = (Map) data.get("target"); + Node target = new Node((String) targetNode.get("parentName"), (String) targetNode.get("parentId")); + if (!sources.isEmpty()) { + for (Map source : sources) { + String parentId = (String) source.get("parentId"); + String parentName = (String) source.get("parentName"); + Node sourceNode = new Node(parentName, parentId); + sourceNode.next = target; + value.add(sourceNode); + nodeMap.put(parentId, sourceNode); + } + } else { + value.add(target); + nodeMap.put((String) targetNode.get("parentId"), target); + } + } + + for (Node node : value) { + Node next = node.next; + if (next != null) { + String id = next.id; + next = nodeMap.get(id); + if (next != null) { + node.next = next; + } + } + } + + HashSet key = new HashSet<>(); + Iterator iterator = value.iterator(); + while (iterator.hasNext()) { + Node node = iterator.next(); + String k = node.key(); + if (key.contains(k)) { + iterator.remove(); + } + key.add(k); + } + + // value + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } +} diff --git a/api/java/MANIFEST-windwos.MF b/api/java/MANIFEST-windwos.MF new file mode 100644 index 0000000..9426094 --- /dev/null +++ b/api/java/MANIFEST-windwos.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Class-Path: lib/fastjson-1.2.47.jar lib/httpclient-4.5.5.jar lib/httpcore-4.4.9.jar lib/httpmime-4.5.6.jar lib/slf4j-api-1.7.25.jar lib/slf4j-log4j12-1.7.25.jar lib/commons-codec-1.10.jar lib/commons-logging-1.2.jar +Main-Class: com.gudusoft.grabit.Runner diff --git a/api/java/MANIFEST.MF b/api/java/MANIFEST.MF new file mode 100644 index 0000000..9426094 --- /dev/null +++ b/api/java/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Class-Path: lib/fastjson-1.2.47.jar lib/httpclient-4.5.5.jar lib/httpcore-4.4.9.jar lib/httpmime-4.5.6.jar lib/slf4j-api-1.7.25.jar lib/slf4j-log4j12-1.7.25.jar lib/commons-codec-1.10.jar lib/commons-logging-1.2.jar +Main-Class: com.gudusoft.grabit.Runner diff --git a/api/java/compile.bat b/api/java/compile.bat new file mode 100644 index 0000000..fd25fb5 --- /dev/null +++ b/api/java/compile.bat @@ -0,0 +1,38 @@ +@ECHO OFF +SETLOCAL enableDelayedExpansion + +SET cur_dir=%CD% +echo %cur_dir% + +SET qddemo=%cur_dir% + +SET qddemo_src=%qddemo%\src +SET qddemo_bin=%qddemo%\lib +SET qddemo_class=%qddemo%\class + +echo %qddemo_class% +echo %qddemo_bin% + +IF EXIST %qddemo_class% RMDIR %qddemo_class% +IF NOT EXIST %qddemo_class% MKDIR %qddemo_class% + +cd %cur_dir% +CD %qddemo_src% +FOR /R %%b IN ( . ) DO ( +IF EXIST %%b/*.java SET JFILES=!JFILES! %%b/*.java +) + +MKDIR %qddemo_class%\lib +XCOPY %qddemo_bin% %qddemo_class%\lib +XCOPY %qddemo%\MANIFEST.MF %qddemo_class% + +cd %cur_dir% + + javac -d %qddemo_class% -encoding utf-8 -cp .;%qddemo_bin%\commons-codec-1.10.jar;%qddemo_bin%\commons-logging-1.2.jar;%qddemo_bin%\fastjson-1.2.47.jar;%qddemo_bin%\httpclient-4.5.5.jar;%qddemo_bin%\httpcore-4.4.9.jar;%qddemo_bin%\httpmime-4.5.6.jar; %JFILES% + +cd %qddemo_class% + jar -cvfm %qddemo%\grabit-java.jar %qddemo%\MANIFEST-windwos.MF * + +echo "successfully" + +pause \ No newline at end of file diff --git a/api/java/compile.sh b/api/java/compile.sh new file mode 100755 index 0000000..fa52eb5 --- /dev/null +++ b/api/java/compile.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +cur_dir=$(pwd) + +function compile(){ + src_dir=$cur_dir/src + bin_dir=$cur_dir/lib + class_dir=$cur_dir/class + + + rm -rf $src_dir/sources.list + find $src_dir -name "*.java" > $src_dir/sources.list + cat $src_dir/sources.list + + + rm -rf $class_dir + mkdir $class_dir + cp $cur_dir/MANIFEST.MF $class_dir + cp -r $cur_dir/lib $class_dir + + javac -d $class_dir -cp .:$bin_dir/fastjson-1.2.47.jar:$bin_dir/commons-codec-1.10.jar:$bin_dir/commons-logging-1.2.jar:$bin_dir/slf4j-api-1.7.25.jar:$bin_dir/slf4j-log4j12-1.7.25.jar:$bin_dir/httpcore-4.4.9.jar:$bin_dir/httpclient-4.5.5.jar:$bin_dir/httpmime-4.5.6.jar -g -sourcepath $src_dir @$src_dir/sources.list + + cd $class_dir + jar -cvfm $cur_dir/grabit-java.jar MANIFEST.MF * +} + +compile +exit 0 \ No newline at end of file diff --git a/api/java/java-data-lineage-overview.png b/api/java/java-data-lineage-overview.png new file mode 100644 index 0000000..b0c1f5b Binary files /dev/null and b/api/java/java-data-lineage-overview.png differ diff --git a/api/java/java-data-lineage-result.json b/api/java/java-data-lineage-result.json new file mode 100644 index 0000000..cb96b4f --- /dev/null +++ b/api/java/java-data-lineage-result.json @@ -0,0 +1,4973 @@ +{ + "code": 203, + "data": { + "mode": "global", + "summary": { + "schema": 4, + "database": 0, + "view": 2, + "mostRelationTables": [ + { + "schema": "SALES", + "table": "SALESPERSON" + }, + { + "schema": "PERSON", + "table": "PERSON" + }, + { + "table": "VIEW1" + } + ], + "column": 34, + "table": 8, + "relation": 33 + }, + "sqlflow": { + "dbvendor": "dbvmssql", + "dbobjs": [ + { + "id": "54", + "schema": "dbo", + "name": "dbo.uspGetEmployeeSales", + "type": "procedure", + "arguments": [], + "coordinates": [ + { + "x": 9, + "y": 18, + "hashCode": "0" + }, + { + "x": 9, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "8", + "schema": "HumanResources", + "name": "HumanResources.Employee", + "alias": "e", + "type": "table", + "columns": [ + { + "id": "113", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 35, + "y": 28, + "hashCode": "0" + }, + { + "x": 35, + "y": 46, + "hashCode": "0" + }, + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "11", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 34, + "y": 6, + "hashCode": "0" + }, + { + "x": 34, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "13", + "schema": "Person", + "name": "Person.Person", + "alias": "p", + "type": "table", + "columns": [ + { + "id": "118", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 35, + "y": 49, + "hashCode": "0" + }, + { + "x": 35, + "y": 67, + "hashCode": "0" + }, + { + "x": 16, + "y": 34, + "hashCode": "0" + }, + { + "x": 16, + "y": 52, + "hashCode": "0" + }, + { + "x": 25, + "y": 34, + "hashCode": "0" + }, + { + "x": 25, + "y": 52, + "hashCode": "0" + }, + { + "x": 51, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 51, + "hashCode": "0" + }, + { + "x": 53, + "y": 26, + "hashCode": "0" + }, + { + "x": 53, + "y": 44, + "hashCode": "0" + } + ] + }, + { + "id": "15", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "119", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 35, + "y": 6, + "hashCode": "0" + }, + { + "x": 35, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "30", + "name": "FactInternetSales", + "alias": "fis", + "type": "table", + "columns": [ + { + "id": "114", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 44, + "y": 5, + "hashCode": "0" + }, + { + "x": 44, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "32", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "33", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "34", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 42, + "y": 6, + "hashCode": "0" + }, + { + "x": 42, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "37", + "name": "DimSalesTerritory", + "alias": "dst", + "type": "table", + "columns": [ + { + "id": "38", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 44, + "y": 27, + "hashCode": "0" + }, + { + "x": 44, + "y": 48, + "hashCode": "0" + } + ] + }, + { + "id": "39", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 43, + "y": 17, + "hashCode": "0" + }, + { + "x": 43, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "56", + "schema": "Sales", + "name": "Sales.SalesPerson", + "alias": "sp", + "type": "table", + "columns": [ + { + "id": "115", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "116", + "name": "SalesYTD", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "111", + "name": "TerritoryID", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 14, + "y": 10, + "hashCode": "0" + }, + { + "x": 14, + "y": 33, + "hashCode": "0" + } + ] + }, + { + "id": "94", + "schema": "Person", + "name": "Person.Address", + "alias": "a", + "type": "table", + "columns": [ + { + "id": "95", + "name": "AddressID", + "coordinates": [ + { + "x": 53, + "y": 12, + "hashCode": "0" + }, + { + "x": 53, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "117", + "name": "PostalCode", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 52, + "y": 16, + "hashCode": "0" + }, + { + "x": 52, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "2", + "schema": "dbo", + "name": "dbo.EmployeeSales", + "type": "table", + "columns": [ + { + "id": "3", + "name": "DataSource", + "coordinates": [ + { + "x": 3, + "y": 3, + "hashCode": "0" + }, + { + "x": 3, + "y": 13, + "hashCode": "0" + } + ] + }, + { + "id": "4", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 4, + "y": 3, + "hashCode": "0" + }, + { + "x": 4, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "5", + "name": "LastName", + "coordinates": [ + { + "x": 5, + "y": 3, + "hashCode": "0" + }, + { + "x": 5, + "y": 11, + "hashCode": "0" + } + ] + }, + { + "id": "6", + "name": "SalesDollars", + "coordinates": [ + { + "x": 6, + "y": 3, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "87", + "name": "DUMMY0", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "1", + "name": "PseudoRows", + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 31, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "109", + "name": "pseudo_table_include_orphan_column", + "type": "pseudoTable", + "columns": [ + { + "id": "110", + "name": "TerritoryID", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 1, + "y": 1, + "hashCode": "0" + }, + { + "x": 1, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "24", + "name": "hiredate_view", + "type": "view", + "columns": [ + { + "id": "25", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "26", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "27", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "28", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "23", + "name": "PseudoRows", + "coordinates": [ + { + "x": 31, + "y": 13, + "hashCode": "0" + }, + { + "x": 31, + "y": 26, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 31, + "y": 13, + "hashCode": "0" + }, + { + "x": 31, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "48", + "name": "view1", + "type": "view", + "columns": [ + { + "id": "49", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "50", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "51", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "52", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "53", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "47", + "name": "PseudoRows", + "coordinates": [ + { + "x": 38, + "y": 13, + "hashCode": "0" + }, + { + "x": 38, + "y": 18, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 38, + "y": 13, + "hashCode": "0" + }, + { + "x": 38, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "id": "18", + "name": "RS-1", + "type": "select_list", + "columns": [ + { + "id": "19", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "20", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "21", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "22", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "17", + "name": "PseudoRows", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "41", + "name": "RS-2", + "type": "select_list", + "columns": [ + { + "id": "42", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "43", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "44", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "45", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "46", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "40", + "name": "PseudoRows", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "66", + "name": "RS-3", + "type": "select_list", + "columns": [ + { + "id": "67", + "name": "PROCEDURE", + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 12, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "69", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + } + ] + }, + { + "id": "70", + "name": "LastName", + "coordinates": [ + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + } + ] + }, + { + "id": "71", + "name": "SalesYTD", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "65", + "name": "PseudoRows", + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "81", + "name": "INSERT-SELECT-1", + "type": "insert-select", + "columns": [ + { + "id": "82", + "name": "SELECT", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "84", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "85", + "name": "LastName", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + }, + { + "id": "86", + "name": "SalesYTD", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + { + "id": "80", + "name": "PseudoRows", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + { + "id": "98", + "name": "RS-4", + "type": "select_list", + "columns": [ + { + "id": "99", + "name": "\"Row Number\"", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 47, + "y": 89, + "hashCode": "0" + } + ] + }, + { + "id": "105", + "name": "LastName", + "coordinates": [ + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "106", + "name": "SalesYTD", + "coordinates": [ + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + } + ] + }, + { + "id": "107", + "name": "PostalCode", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "97", + "name": "PseudoRows", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + } + ], + "relations": [ + { + "id": "4101", + "type": "fdd", + "effectType": "select", + "target": { + "id": "107", + "column": "PostalCode", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "117", + "column": "PostalCode", + "parentId": "94", + "parentName": "Person.Address", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4102", + "type": "fdd", + "effectType": "select", + "target": { + "id": "69", + "column": "BusinessEntityID", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "115", + "column": "BusinessEntityID", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4103", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "52", + "column": "SalesTerritoryKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "45", + "column": "SalesTerritoryKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4104", + "type": "fdd", + "effectType": "select", + "target": { + "id": "21", + "column": "BusinessEntityID", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "113", + "column": "BusinessEntityID", + "parentId": "8", + "parentName": "HumanResources.Employee", + "coordinates": [ + { + "x": 35, + "y": 28, + "hashCode": "0" + }, + { + "x": 35, + "y": 46, + "hashCode": "0" + }, + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4105", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "26", + "column": "LastName", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "20", + "column": "LastName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4106", + "type": "fdd", + "effectType": "select", + "target": { + "id": "20", + "column": "LastName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4107", + "type": "fdd", + "effectType": "select", + "target": { + "id": "99", + "column": "\"Row Number\"", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 47, + "y": 89, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "117", + "column": "PostalCode", + "parentId": "94", + "parentName": "Person.Address", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4108", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "28", + "column": "HireDate", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "22", + "column": "HireDate", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4109", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "27", + "column": "BusinessEntityID", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "21", + "column": "BusinessEntityID", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4110", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22", + "column": "HireDate", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "11", + "column": "HireDate", + "parentId": "8", + "parentName": "HumanResources.Employee", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4111", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "4", + "column": "BusinessEntityID", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 4, + "y": 3, + "hashCode": "0" + }, + { + "x": 4, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "84", + "column": "BusinessEntityID", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4112", + "type": "fdd", + "effectType": "select", + "target": { + "id": "106", + "column": "SalesYTD", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4113", + "type": "fdd", + "effectType": "select", + "target": { + "id": "86", + "column": "SalesYTD", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4114", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "51", + "column": "OrderDateKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "44", + "column": "OrderDateKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4115", + "type": "fdd", + "effectType": "select", + "target": { + "id": "70", + "column": "LastName", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4116", + "type": "fdd", + "effectType": "select", + "target": { + "id": "71", + "column": "SalesYTD", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4117", + "type": "fdd", + "effectType": "select", + "target": { + "id": "43", + "column": "ProductKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "33", + "column": "ProductKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4118", + "type": "fdd", + "effectType": "select", + "target": { + "id": "85", + "column": "LastName", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4119", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "50", + "column": "ProductKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "43", + "column": "ProductKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4120", + "type": "fdd", + "effectType": "select", + "target": { + "id": "44", + "column": "OrderDateKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "34", + "column": "OrderDateKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4121", + "type": "fdd", + "effectType": "select", + "target": { + "id": "42", + "column": "CustomerKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "32", + "column": "CustomerKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4122", + "type": "fdd", + "effectType": "select", + "target": { + "id": "84", + "column": "BusinessEntityID", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "115", + "column": "BusinessEntityID", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4123", + "type": "fdd", + "effectType": "select", + "target": { + "id": "19", + "column": "FirstName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "15", + "column": "FirstName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4124", + "type": "fdd", + "effectType": "select", + "target": { + "id": "105", + "column": "LastName", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4125", + "type": "fdd", + "effectType": "select", + "target": { + "id": "45", + "column": "SalesTerritoryKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "114", + "column": "SalesTerritoryKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 44, + "y": 5, + "hashCode": "0" + }, + { + "x": 44, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4126", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "49", + "column": "CustomerKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "42", + "column": "CustomerKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4127", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "5", + "column": "LastName", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 5, + "y": 3, + "hashCode": "0" + }, + { + "x": 5, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "85", + "column": "LastName", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4128", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "6", + "column": "SalesDollars", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 6, + "y": 3, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "86", + "column": "SalesYTD", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4129", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "25", + "column": "FirstName", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "19", + "column": "FirstName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4130", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "87", + "column": "DUMMY0", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "82", + "column": "SELECT", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4131", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "53", + "column": "SalesTerritoryRegion", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "46", + "column": "SalesTerritoryRegion", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4132", + "type": "fdd", + "effectType": "select", + "target": { + "id": "46", + "column": "SalesTerritoryRegion", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "39", + "column": "SalesTerritoryRegion", + "parentId": "37", + "parentName": "DimSalesTerritory", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ] + } + ], + "errors": [ + { + "errorMessage": "find orphan column(10500) near: PostalCode(47,39)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: SalesYTD(47,59)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: TerritoryID(54,7)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: SalesYTD(55,9)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: PostalCode(56,10)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 56, + "y": 10, + "hashCode": "0" + }, + { + "x": 56, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "Link orphan column [TerritoryID] to the first table [Sales.SalesPerson s]", + "errorType": "LinkOrphanColumn", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ] + }, + "graph": { + "elements": { + "tables": [ + { + "columns": [ + { + "height": 16, + "id": "n0::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -625.07874 + }, + { + "height": 16, + "id": "n0::n1", + "label": { + "content": "HIREDATE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 76, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -609.07874 + } + ], + "height": 57.96875, + "id": "n0", + "label": { + "content": "HumanResources.Employee", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 178, + "x": -8, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -647.0475 + }, + { + "columns": [ + { + "height": 16, + "id": "n1::n0", + "label": { + "content": "FIRSTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 84, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -537.1097 + }, + { + "height": 16, + "id": "n1::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -521.1097 + } + ], + "height": 57.96875, + "id": "n1", + "label": { + "content": "Person.Person", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -559.0784 + }, + { + "columns": [ + { + "height": 16, + "id": "n2::n0", + "label": { + "content": "SALESTERRITORYKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 141, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -78.00015 + }, + { + "height": 16, + "id": "n2::n1", + "label": { + "content": "CUSTOMERKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 105, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -62.00015 + }, + { + "height": 16, + "id": "n2::n2", + "label": { + "content": "PRODUCTKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -46.00015 + }, + { + "height": 16, + "id": "n2::n3", + "label": { + "content": "ORDERDATEKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 112, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -30.000149 + } + ], + "height": 89.96875, + "id": "n2", + "label": { + "content": "FactInternetSales", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -99.9689 + }, + { + "columns": [ + { + "height": 16, + "id": "n3::n0", + "label": { + "content": "SALESTERRITORYREGION", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 165, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -149.9692 + } + ], + "height": 41.96875, + "id": "n3", + "label": { + "content": "DimSalesTerritory", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -171.93794 + }, + { + "columns": [ + { + "height": 16, + "id": "n4::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -362.57196 + }, + { + "height": 16, + "id": "n4::n1", + "label": { + "content": "SALESYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 77, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -346.57196 + } + ], + "height": 57.96875, + "id": "n4", + "label": { + "content": "Sales.SalesPerson", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -384.5407 + }, + { + "columns": [ + { + "height": 16, + "id": "n5::n0", + "label": { + "content": "POSTALCODE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -221.93825 + } + ], + "height": 41.96875, + "id": "n5", + "label": { + "content": "Person.Address", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -243.907 + }, + { + "columns": [ + { + "height": 16, + "id": "n6::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -495.37158 + }, + { + "height": 16, + "id": "n6::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -479.37158 + }, + { + "height": 16, + "id": "n6::n2", + "label": { + "content": "SALESDOLLARS", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 108, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -463.37158 + }, + { + "height": 16, + "id": "n6::n3", + "label": { + "content": "DUMMY0", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 69, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -447.37158 + } + ], + "height": 89.96875, + "id": "n6", + "label": { + "content": "dbo.EmployeeSales", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -517.34033 + }, + { + "columns": [ + { + "height": 16, + "id": "n7::n0", + "label": { + "content": "FIRSTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 84, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -615.34064 + }, + { + "height": 16, + "id": "n7::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -599.34064 + }, + { + "height": 16, + "id": "n7::n2", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -583.34064 + }, + { + "height": 16, + "id": "n7::n3", + "label": { + "content": "HIREDATE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 76, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -567.34064 + } + ], + "height": 89.96875, + "id": "n7", + "label": { + "content": "hiredate_view", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -637.3094 + }, + { + "columns": [ + { + "height": 16, + "id": "n8::n0", + "label": { + "content": "CUSTOMERKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 105, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -97.24618 + }, + { + "height": 16, + "id": "n8::n1", + "label": { + "content": "PRODUCTKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -81.24618 + }, + { + "height": 16, + "id": "n8::n2", + "label": { + "content": "ORDERDATEKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 112, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -65.24618 + }, + { + "height": 16, + "id": "n8::n3", + "label": { + "content": "SALESTERRITORYKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 141, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -49.24618 + }, + { + "height": 16, + "id": "n8::n4", + "label": { + "content": "SALESTERRITORYREGION", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 165, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -33.24618 + } + ], + "height": 105.96875, + "id": "n8", + "label": { + "content": "view1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -119.21493 + }, + { + "columns": [ + { + "height": 16, + "id": "n9::n0", + "label": { + "content": "FirstName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 79, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -615.34064 + }, + { + "height": 16, + "id": "n9::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -599.34064 + }, + { + "height": 16, + "id": "n9::n2", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -583.34064 + }, + { + "height": 16, + "id": "n9::n3", + "label": { + "content": "HireDate", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 71, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -567.34064 + } + ], + "height": 89.96875, + "id": "n9", + "label": { + "content": "RS-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -637.3094 + }, + { + "columns": [ + { + "height": 16, + "id": "n10::n0", + "label": { + "content": "CustomerKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 97, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -97.24618 + }, + { + "height": 16, + "id": "n10::n1", + "label": { + "content": "ProductKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 86, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -81.24618 + }, + { + "height": 16, + "id": "n10::n2", + "label": { + "content": "OrderDateKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 102, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -65.24618 + }, + { + "height": 16, + "id": "n10::n3", + "label": { + "content": "SalesTerritoryKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 123, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -49.24618 + }, + { + "height": 16, + "id": "n10::n4", + "label": { + "content": "SalesTerritoryRegion", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 145, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -33.24618 + } + ], + "height": 105.96875, + "id": "n10", + "label": { + "content": "RS-2", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -119.21493 + }, + { + "columns": [ + { + "height": 16, + "id": "n11::n0", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -375.40253 + }, + { + "height": 16, + "id": "n11::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -359.40253 + }, + { + "height": 16, + "id": "n11::n2", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -343.40253 + } + ], + "height": 73.96875, + "id": "n11", + "label": { + "content": "RS-3", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -397.37128 + }, + { + "columns": [ + { + "height": 16, + "id": "n12::n0", + "label": { + "content": "SELECT", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -495.37158 + }, + { + "height": 16, + "id": "n12::n1", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -479.37158 + }, + { + "height": 16, + "id": "n12::n2", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -463.37158 + }, + { + "height": 16, + "id": "n12::n3", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -447.37158 + } + ], + "height": 89.96875, + "id": "n12", + "label": { + "content": "INSERT-SELECT-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -517.34033 + }, + { + "columns": [ + { + "height": 16, + "id": "n13::n0", + "label": { + "content": "\"Row Number\"", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 104, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -271.43347 + }, + { + "height": 16, + "id": "n13::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -255.43349 + }, + { + "height": 16, + "id": "n13::n2", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -239.43349 + }, + { + "height": 16, + "id": "n13::n3", + "label": { + "content": "PostalCode", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 87, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -223.43349 + } + ], + "height": 89.96875, + "id": "n13", + "label": { + "content": "RS-4", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -293.40222 + } + ], + "edges": [ + { + "id": "e0", + "sourceId": "n4::n0", + "targetId": "n12::n1" + }, + { + "id": "e1", + "sourceId": "n9::n1", + "targetId": "n7::n1" + }, + { + "id": "e2", + "sourceId": "n2::n3", + "targetId": "n10::n2" + }, + { + "id": "e3", + "sourceId": "n9::n3", + "targetId": "n7::n3" + }, + { + "id": "e4", + "sourceId": "n1::n1", + "targetId": "n13::n1" + }, + { + "id": "e5", + "sourceId": "n1::n1", + "targetId": "n12::n2" + }, + { + "id": "e6", + "sourceId": "n10::n4", + "targetId": "n8::n4" + }, + { + "id": "e7", + "sourceId": "n5::n0", + "targetId": "n13::n3" + }, + { + "id": "e8", + "sourceId": "n2::n2", + "targetId": "n10::n1" + }, + { + "id": "e9", + "sourceId": "n1::n1", + "targetId": "n11::n1" + }, + { + "id": "e10", + "sourceId": "n12::n3", + "targetId": "n6::n2" + }, + { + "id": "e11", + "sourceId": "n2::n1", + "targetId": "n10::n0" + }, + { + "id": "e12", + "sourceId": "n4::n1", + "targetId": "n11::n2" + }, + { + "id": "e13", + "sourceId": "n12::n2", + "targetId": "n6::n1" + }, + { + "id": "e14", + "sourceId": "n1::n1", + "targetId": "n9::n1" + }, + { + "id": "e15", + "sourceId": "n12::n1", + "targetId": "n6::n0" + }, + { + "id": "e16", + "sourceId": "n1::n0", + "targetId": "n9::n0" + }, + { + "id": "e17", + "sourceId": "n9::n2", + "targetId": "n7::n2" + }, + { + "id": "e18", + "sourceId": "n4::n1", + "targetId": "n12::n3" + }, + { + "id": "e19", + "sourceId": "n0::n1", + "targetId": "n9::n3" + }, + { + "id": "e20", + "sourceId": "n9::n0", + "targetId": "n7::n0" + }, + { + "id": "e21", + "sourceId": "n3::n0", + "targetId": "n10::n4" + }, + { + "id": "e22", + "sourceId": "n12::n0", + "targetId": "n6::n3" + }, + { + "id": "e23", + "sourceId": "n2::n0", + "targetId": "n10::n3" + }, + { + "id": "e24", + "sourceId": "n0::n0", + "targetId": "n9::n2" + }, + { + "id": "e25", + "sourceId": "n10::n2", + "targetId": "n8::n2" + }, + { + "id": "e26", + "sourceId": "n4::n1", + "targetId": "n13::n2" + }, + { + "id": "e27", + "sourceId": "n4::n1", + "targetId": "n13::n0" + }, + { + "id": "e28", + "sourceId": "n5::n0", + "targetId": "n13::n0" + }, + { + "id": "e29", + "sourceId": "n10::n1", + "targetId": "n8::n1" + }, + { + "id": "e30", + "sourceId": "n10::n0", + "targetId": "n8::n0" + }, + { + "id": "e31", + "sourceId": "n10::n3", + "targetId": "n8::n3" + }, + { + "id": "e32", + "sourceId": "n4::n0", + "targetId": "n11::n0" + } + ] + }, + "tooltip": { + "pseudo_tab...rphan_column": "pseudo_table_include_orphan_column" + }, + "relationIdMap": { + "e0": "fdd", + "e1": "fdd", + "e2": "fdd", + "e3": "fdd", + "e4": "fdd", + "e5": "fdd", + "e6": "fdd", + "e7": "fdd", + "e8": "fdd", + "e9": "fdd", + "e10": "fdd", + "e11": "fdd", + "e12": "fdd", + "e13": "fdd", + "e14": "fdd", + "e15": "fdd", + "e16": "fdd", + "e17": "fdd", + "e18": "fdd", + "e19": "fdd", + "e20": "fdd", + "e21": "fdd", + "e22": "fdd", + "e23": "fdd", + "e24": "fdd", + "e25": "fdd", + "e26": "fdd", + "e27": "fdd", + "e28": "fdd", + "e29": "fdd", + "e30": "fdd", + "e31": "fdd", + "e32": "fdd" + }, + "listIdMap": { + "n0": [ + "8" + ], + "n0::n0": [ + "113" + ], + "n0::n1": [ + "11" + ], + "n1": [ + "13" + ], + "n1::n0": [ + "15" + ], + "n1::n1": [ + "119" + ], + "n2": [ + "30" + ], + "n2::n0": [ + "114" + ], + "n2::n1": [ + "32" + ], + "n2::n2": [ + "33" + ], + "n2::n3": [ + "34" + ], + "n3": [ + "37" + ], + "n3::n0": [ + "39" + ], + "n4": [ + "56" + ], + "n4::n0": [ + "115" + ], + "n4::n1": [ + "116" + ], + "n5": [ + "94" + ], + "n5::n0": [ + "117" + ], + "n6": [ + "2" + ], + "n6::n0": [ + "4" + ], + "n6::n1": [ + "5" + ], + "n6::n2": [ + "6" + ], + "n6::n3": [ + "87" + ], + "n7": [ + "24" + ], + "n7::n0": [ + "25" + ], + "n7::n1": [ + "26" + ], + "n7::n2": [ + "27" + ], + "n7::n3": [ + "28" + ], + "n8": [ + "48" + ], + "n8::n0": [ + "49" + ], + "n8::n1": [ + "50" + ], + "n8::n2": [ + "51" + ], + "n8::n3": [ + "52" + ], + "n8::n4": [ + "53" + ], + "n9": [ + "18" + ], + "n9::n0": [ + "19" + ], + "n9::n1": [ + "20" + ], + "n9::n2": [ + "21" + ], + "n9::n3": [ + "22" + ], + "n10": [ + "41" + ], + "n10::n0": [ + "42" + ], + "n10::n1": [ + "43" + ], + "n10::n2": [ + "44" + ], + "n10::n3": [ + "45" + ], + "n10::n4": [ + "46" + ], + "n11": [ + "66" + ], + "n11::n0": [ + "69" + ], + "n11::n1": [ + "70" + ], + "n11::n2": [ + "71" + ], + "n12": [ + "81" + ], + "n12::n0": [ + "82" + ], + "n12::n1": [ + "84" + ], + "n12::n2": [ + "85" + ], + "n12::n3": [ + "86" + ], + "n13": [ + "98" + ], + "n13::n0": [ + "99" + ], + "n13::n1": [ + "105" + ], + "n13::n2": [ + "106" + ], + "n13::n3": [ + "107" + ] + } + } + }, + "error": "find orphan column(10500) near: PostalCode(47,39)\r\nfind orphan column(10500) near: SalesYTD(47,59)\r\nfind orphan column(10500) near: TerritoryID(54,7)\r\nfind orphan column(10500) near: SalesYTD(55,9)\r\nfind orphan column(10500) near: PostalCode(56,10)\r\nLink orphan column [TerritoryID] to the first table [Sales.SalesPerson s]", + "sessionId": "ec3938ae68a8a468d729e6c426108e701e78079cf070a385acf7cd899d41058c_1615189437331" +} diff --git a/api/java/java-data-lineage-sqlserver.sql b/api/java/java-data-lineage-sqlserver.sql new file mode 100644 index 0000000..9c34039 --- /dev/null +++ b/api/java/java-data-lineage-sqlserver.sql @@ -0,0 +1,56 @@ +-- sql server sample sql +CREATE TABLE dbo.EmployeeSales +( DataSource varchar(20) NOT NULL, + BusinessEntityID varchar(11) NOT NULL, + LastName varchar(40) NOT NULL, + SalesDollars money NOT NULL +); +GO +CREATE PROCEDURE dbo.uspGetEmployeeSales +AS + SET NOCOUNT ON; + SELECT 'PROCEDURE', sp.BusinessEntityID, c.LastName, + sp.SalesYTD + FROM Sales.SalesPerson AS sp + INNER JOIN Person.Person AS c + ON sp.BusinessEntityID = c.BusinessEntityID + WHERE sp.BusinessEntityID LIKE '2%' + ORDER BY sp.BusinessEntityID, c.LastName; +GO +--INSERT...SELECT example +INSERT INTO dbo.EmployeeSales + SELECT 'SELECT', sp.BusinessEntityID, c.LastName, sp.SalesYTD + FROM Sales.SalesPerson AS sp + INNER JOIN Person.Person AS c + ON sp.BusinessEntityID = c.BusinessEntityID + WHERE sp.BusinessEntityID LIKE '2%' + ORDER BY sp.BusinessEntityID, c.LastName; +GO + + +CREATE VIEW hiredate_view +AS +SELECT p.FirstName, p.LastName, e.BusinessEntityID, e.HireDate +FROM HumanResources.Employee e +JOIN Person.Person AS p ON e.BusinessEntityID = p.BusinessEntityID ; +GO + +CREATE VIEW view1 +AS +SELECT fis.CustomerKey, fis.ProductKey, fis.OrderDateKey, + fis.SalesTerritoryKey, dst.SalesTerritoryRegion +FROM FactInternetSales AS fis +LEFT OUTER JOIN DimSalesTerritory AS dst +ON (fis.SalesTerritoryKey=dst.SalesTerritoryKey); + +GO +SELECT ROW_NUMBER() OVER(PARTITION BY PostalCode ORDER BY SalesYTD DESC) AS "Row Number", + p.LastName, s.SalesYTD, a.PostalCode +FROM Sales.SalesPerson AS s + INNER JOIN Person.Person AS p + ON s.BusinessEntityID = p.BusinessEntityID + INNER JOIN Person.Address AS a + ON a.AddressID = p.BusinessEntityID +WHERE TerritoryID IS NOT NULL + AND SalesYTD <> 0 +ORDER BY PostalCode; diff --git a/api/java/java-data-lineage.png b/api/java/java-data-lineage.png new file mode 100644 index 0000000..14955be Binary files /dev/null and b/api/java/java-data-lineage.png differ diff --git a/api/java/readme.md b/api/java/readme.md new file mode 100644 index 0000000..abf036c --- /dev/null +++ b/api/java/readme.md @@ -0,0 +1,156 @@ +## JAVA Data lineage: using the SQLFlow REST API (Advanced) + +This article illustrates how to discover the data lineage using JAVA and the SQLFlow REST API. + +By using the SQLFlow REST API, you can code in JAVA to discover the data lineage in SQL scripts +and get the result in an actionable diagram, json, csv or graphml format. + +You can integerate the JAVA code provided here into your own project and add the powerful +data lineage analsysis capability instantly. + +### 1. interactive data lineage visualizations +![JAVA Data lineage](java-data-lineage.png) + +### 2. [Data lineage in JSON format](java-data-lineage-result.json) + +### 3. Data lineage in CSV, graphml format + + +## Prerequisites +- [SQLFlow Cloud Server or on-premise version](https://github.com/sqlparser/sqlflow_public/tree/master/api#prerequisites) + +- Java 8 or higher version must be installed and configured correctly. + +- setup the PATH like this: (Please change the JAVA_HOME according to your environment) +``` +export JAVA_HOME=/usr/lib/jvm/default-java + +export PATH=$JAVA_HOME/bin:$PATH +``` + +- compile and build `grabit-java.jar` + +**mac&linux** +``` +chmod 777 compile.sh + +./compile.sh +``` + +**windows** + +``` +compile.bat +``` + +### Usage + +```` +java -jar grabit-java.jar /s server /p port /u userId /k userSecret /t databaseType /f path_to_config_file /r resultType + +eg: + java -jar grabit-java.jar /u 'auth0|xxx' /k cab9712c45189014a94a8b7aceeef7a3db504be58e18cd3686f3bbefd078ef4d /s https://api.gudusoft.com /t oracle /f demo.sql /r 1 + +note: + If the parameter string contains symbols like "|" , it must be included in a single quotes (' ') or double quotes on windows (" ") +```` + +Example: + +1. Connect to the SQLFlow Cloud Server +``` +java -jar grabit-java.jar /s https://api.gudusoft.com /u 'YOUR_USER_ID' /k YOUR_SECRET_KEY /t sqlserver /f java-data-lineage-sqlserver.sql /r 1 +``` + +2. Connect to the SQLFlow on-premise +This will discover data lineage by analyzing the `java-data-lineage-sqlserver.sql` file. You may also specify a zip file which includes lots of SQL files. +``` +java -jar grabit-java.jar /s http://127.0.0.1 /p 8081 /u 'gudu|0123456789' /t sqlserver /f java-data-lineage-sqlserver.sql /r 1 +``` + +This will discover data lineage by analyzing all SQL files under `sqlfiles` directory. +``` +java -jar grabit-java.jar /s http://127.0.0.1 /p 8081 /u 'gudu|0123456789' /t mysql /f sqlfiles /r 1 +``` + +After execution, view the `logs/graibt.log` file for the detailed information. + +If the log prints a **submit the job to sqlflow successful**. +Then it is proved that the upload to SQLFlow has been successful. + +Log in to the SQLFlow website to view the newly analyzed results. +In the `Job List`, you can view the analysis results of the currently submitted tasks. + +### Parameters + +- **path_to_config_file** + +This can be a single SQL file, a zip file including multiple SQL files, or a directory including lots of SQL files. + +- **server** + +Usually, it is the IP address of [the SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +installed on your owner servers such as `127.0.0.1` or `http://127.0.0.1` + +You may set the value to `https://api.gudusoft.com` if you like to send your SQL script to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com) to get the data lineage result. + +- **port** + +The default value is `8081` if you connect to your SQLFlow on-premise server. + +However, if you setup the nginx reverse proxy in the nginx configuration file like this: +``` + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } +``` +Then, keep the value of `serverPort` empty and set `server` to the value like this: `http://127.0.0.1/api`. + +>Please keep this value empty if you connect to the SQLFlow Cloud Server by specifying the `https://api.gudusoft.com` +in the `server` + > +- **userId, userSecret** + +This is the user id that is used to connect to the SQLFlow server. +Always set this value to `gudu|0123456789` and keep `userSecret` empty if you use the SQLFlow on-premise version. + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + + +- **databaseType** + +This parameter specifies the database dialect of the SQL scripts that the SQLFlow has analyzed. + +```txt + access,bigquery,couchbase,dax,db2,greenplum,hana,hive,impala,informix,mdx,mssql, + sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake, + sybase,teradata,soql,vertica +``` + +- **resultType** + +When you submit SQL script to the SQLFlow server, A job is created on the SQLFlow server +and you can always see the graphic data lineage result via the browser, + + +Even better, This demo will fetch the data lineage back to the directory where the demo is running. +Those data lineage results are stored in the `data/result/` directory. + +This parameter specifies which kind of format is used to save the data lineage result. + +Available values for this parameter: +- 1: JSON, data lineage result in JSON. +- 2: CSV, data lineage result in CSV format. +- 3: diagram, in graphml format that can be viewed by yEd. + +### SQLFlow REST API +Please check here for the detailed information about the [SQLFlow REST API](https://github.com/sqlparser/sqlflow_public/tree/master/api/sqlflow_api.md) diff --git a/api/java/src/main/java/com/gudusoft/grabit/DateUtil.java b/api/java/src/main/java/com/gudusoft/grabit/DateUtil.java new file mode 100644 index 0000000..696d4b8 --- /dev/null +++ b/api/java/src/main/java/com/gudusoft/grabit/DateUtil.java @@ -0,0 +1,28 @@ +package com.gudusoft.grabit; + +import java.text.SimpleDateFormat; +import java.util.Date; + +public class DateUtil { + public DateUtil() { + } + + public static String format(Date date) { + return format(date, "yyyyMMdd"); + } + + public static String format(Date date, String pattern) { + if (date != null) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + return df.format(date); + } else { + return null; + } + } + + public static String timeStamp2Date(Long seconds) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + return sdf.format(new Date(seconds)); + } + +} diff --git a/api/java/src/main/java/com/gudusoft/grabit/FileUtil.java b/api/java/src/main/java/com/gudusoft/grabit/FileUtil.java new file mode 100644 index 0000000..932c84b --- /dev/null +++ b/api/java/src/main/java/com/gudusoft/grabit/FileUtil.java @@ -0,0 +1,89 @@ +package com.gudusoft.grabit; + +import java.io.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class FileUtil { + + private static final int BUFFER_SIZE = 10 * 1024 * 1024; + + private FileUtil() { + } + + public static void mkFile(String filePath) throws IOException { + File testFile = new File(filePath); + File fileParent = testFile.getParentFile(); + if (!fileParent.exists()) { + fileParent.mkdirs(); + } + if (!testFile.exists()) { + testFile.createNewFile(); + } + } + + public static void toZip(String srcDir, OutputStream out, boolean KeepDirStructure) + throws RuntimeException { + ZipOutputStream zos = null; + try { + zos = new ZipOutputStream(out); + File sourceFile = new File(srcDir); + compress(sourceFile, zos, sourceFile.getName(), KeepDirStructure); + } catch (Exception e) { + throw new RuntimeException("zip error from ZipUtils", e); + } finally { + if (zos != null) { + try { + zos.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + } + + private static void compress(File sourceFile, ZipOutputStream zos, String name, + boolean KeepDirStructure) throws Exception { + byte[] buf = new byte[BUFFER_SIZE]; + if (sourceFile.isFile()) { + zos.putNextEntry(new ZipEntry(name)); + int len; + FileInputStream in = new FileInputStream(sourceFile); + while ((len = in.read(buf)) != -1) { + zos.write(buf, 0, len); + } + zos.closeEntry(); + in.close(); + } else { + File[] listFiles = sourceFile.listFiles(); + if (listFiles == null || listFiles.length == 0) { + if (KeepDirStructure) { + zos.putNextEntry(new ZipEntry(name + "/")); + zos.closeEntry(); + } + + } else { + for (File file : listFiles) { + if (KeepDirStructure) { + compress(file, zos, name + "/" + file.getName(), KeepDirStructure); + } else { + compress(file, zos, file.getName(), KeepDirStructure); + } + } + } + } + } + + public static OutputStream outStream(String path) throws IOException { + FileOutputStream fileOutputStream; + try { + fileOutputStream = new FileOutputStream(path); + } catch (Exception ex) { + mkFile(path); + fileOutputStream = new FileOutputStream(path); + } + return fileOutputStream; + } + +} diff --git a/api/java/src/main/java/com/gudusoft/grabit/Runner.java b/api/java/src/main/java/com/gudusoft/grabit/Runner.java new file mode 100644 index 0000000..8f97d61 --- /dev/null +++ b/api/java/src/main/java/com/gudusoft/grabit/Runner.java @@ -0,0 +1,182 @@ +package com.gudusoft.grabit; + +import com.alibaba.fastjson.JSONObject; +import com.gudusoft.grabit.SqlFlowUtil; +import com.gudusoft.grabit.DateUtil; +import com.gudusoft.grabit.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + + +public class Runner { + + + public static void main(String[] args) throws IOException { + if (args.length < 2) { + System.err.println("please enter the correct parameters."); + return; + } + + List argList = Arrays.asList(args); + matchParam("/f", argList); + String fileVal = detectParam("/f", args, argList); + File file = new File(fileVal); + if (!file.exists()) { + System.err.println("{} is not exist." + file); + return; + } + + matchParam("/s", argList); + String server = detectParam("/s", args, argList); + if (!server.startsWith("http") && !server.startsWith("https")) { + server = "http://" + server; + } + if (server.endsWith(File.separator)) { + server = server.substring(0, server.length() - 1); + } + + if (argList.contains("/p") && argList.size() > argList.indexOf("/p") + 1) { + server = server + ":" + detectParam("/p", args, argList); + } + + matchParam("/u", argList); + String userId = detectParam("/u", args, argList).replace("'", ""); + + String userSecret = ""; + if (argList.contains("/k") && argList.size() > argList.indexOf("/k") + 1) { + userSecret = detectParam("/k", args, argList); + } + + String databaseType = "dbvoracle"; + if (argList.contains("/t") && argList.size() > argList.indexOf("/t") + 1) { + databaseType = "dbv" + detectParam("/t", args, argList); + if ("dbvsqlserver".equalsIgnoreCase(databaseType)) { + databaseType = "dbvmssql"; + } + } + + int resultType = 1; + if (argList.contains("/r") && argList.size() > argList.indexOf("/r") + 1) { + resultType = Integer.parseInt(detectParam("/r", args, argList)); + } + + System.out.println("================= run start grabit =================="); + run(file, server, userId, userSecret, databaseType, resultType); + System.out.println("================= run end grabit =================="); + } + + private static void run(File file, String server, String userId, String userSecret, String databaseType, Integer resultType) throws IOException { + String tokenUrl = String.format("%s/gspLive_backend/user/generateToken", server); + String token = SqlFlowUtil.getToken(tokenUrl, userId, userSecret, 0); + if ("".equals(token)) { + System.err.println("connection to sqlflow failed."); + System.exit(1); + } + + String path = ""; + if (file.isDirectory()) { + path = file.getPath() + ".zip"; + FileUtil.toZip(file.getPath(), FileUtil.outStream(path), true); + } else if (file.isFile()) { + path = file.getPath(); + } + + String submitUrl = String.format("%s/gspLive_backend/sqlflow/job/submitUserJob", server); + final String taskName = DateUtil.format(new Date()) + "_" + System.currentTimeMillis(); + String result = SqlFlowUtil.submitJob(path, submitUrl, + databaseType, + userId, token, + taskName); + JSONObject object = JSONObject.parseObject(result); + if (null != object) { + Integer code = object.getInteger("code"); + if (code == 200) { + JSONObject data = object.getJSONObject("data"); + System.out.println("submit job to sqlflow successful. SQLFlow is being analyzed..."); + String jobId = data.getString("jobId"); + + String jsonJobUrl = String.format("%s/gspLive_backend/sqlflow/job/displayUserJobSummary", server); + while (true) { + String statusRs = SqlFlowUtil.getStatus(jsonJobUrl, userId, token, jobId); + JSONObject statusObj = JSONObject.parseObject(statusRs); + if (null != statusObj) { + if (statusObj.getInteger("code") == 200) { + JSONObject val = statusObj.getJSONObject("data"); + String status = val.getString("status"); + if ("success".equals(status) || "partial_success".equals(status)) { + System.out.println("sqlflow analyze successful."); + break; + } + if ("fail".equals(status)) { + System.err.println(val.getString("errorMessage")); + System.exit(1); + } + } + } + } + + String rsUrl = ""; + String downLoadPath = ""; + String rootPath = "data" + File.separator + "result" + File.separator + DateUtil.timeStamp2Date(System.currentTimeMillis()) + "_" + jobId; + switch (resultType) { + case 1: + rsUrl = String.format("%s/gspLive_backend/sqlflow/job/exportLineageAsJson", server); + downLoadPath = rootPath + "_json.json"; + break; + case 2: + rsUrl = String.format("%s/gspLive_backend/sqlflow/job/exportLineageAsCsv", server); + downLoadPath = rootPath + "_csv.csv"; + break; + case 3: + rsUrl = String.format("%s/gspLive_backend/sqlflow/job/exportLineageAsGraphml", server); + downLoadPath = rootPath + "_graphml.graphml"; + break; + default: + break; + } + + SqlFlowUtil.ExportLineageReq request = new SqlFlowUtil.ExportLineageReq(); + request.setToken(token); + request.setJobId(jobId); + request.setTableToTable(true); + request.setUserId(userId); + request.setUrl(rsUrl); + request.setDownloadFilePath(downLoadPath); + + System.out.println("start export result from sqlflow."); + result = SqlFlowUtil.exportLineage(request); + if (!result.contains("success")) { + System.err.println("export json result failed"); + System.exit(1); + } + + System.out.println("export json result successful,downloaded file path is {}" + downLoadPath); + } else { + System.err.println("submit job to sqlflow failed."); + System.exit(1); + } + } + } + + private static String detectParam(String param, String[] args, List argList) { + try { + return args[argList.indexOf(param) + 1]; + } catch (Exception e) { + System.err.println("Please enter the correct parameters."); + System.exit(1); + } + return null; + } + + private static void matchParam(String param, List argList) { + if (!argList.contains(param) || argList.size() <= argList.indexOf(param) + 1) { + System.err.println("{} parameter is required." + param); + System.exit(1); + } + } + +} diff --git a/api/java/src/main/java/com/gudusoft/grabit/SqlFlowUtil.java b/api/java/src/main/java/com/gudusoft/grabit/SqlFlowUtil.java new file mode 100644 index 0000000..2abfe36 --- /dev/null +++ b/api/java/src/main/java/com/gudusoft/grabit/SqlFlowUtil.java @@ -0,0 +1,227 @@ +package com.gudusoft.grabit; + +import com.alibaba.fastjson.JSONObject; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; + +import java.io.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SqlFlowUtil { + + private static String token = ""; + + private SqlFlowUtil() { + } + + public static String getToken(String url, String userId, + String secretKey, Integer flag) { + try { + System.out.println("start get token from sqlflow."); + Map param = new HashMap<>(); + param.put("secretKey", secretKey); + param.put("userId", userId); + if ("gudu|0123456789".equals(userId)) { + return "token"; + } + String result = doPost(url, param); + JSONObject object = JSONObject.parseObject(result); + if ("200".equals(object.getString("code"))) { + token = object.getString("token"); + System.out.println("get token from sqlflow successful."); + return token; + } + return ""; + } catch (Exception e) { + if (flag == 0) { + if (url.startsWith("http:")) { + url = url.replace("http", "https"); + } + return getToken(url, userId, + secretKey, 1); + } + if (flag == 1) { + System.err.println("get token from sqlflow failed."); + } + return token; + } + } + + public static String submitJob(String filePath, + String url, + String dbVendor, + String userId, + String token, + String jobName) throws IOException { + System.out.println("start submit job to sqlflow."); + CloseableHttpClient httpClient = HttpClients.createDefault(); + HttpPost uploadFile = new HttpPost(url); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("dbvendor", dbVendor, ContentType.TEXT_PLAIN); + builder.addTextBody("jobName", jobName, ContentType.TEXT_PLAIN); + builder.addTextBody("token", token, ContentType.TEXT_PLAIN); + builder.addTextBody("userId", userId, ContentType.TEXT_PLAIN); + File f = new File(filePath); + builder.addBinaryBody("sqlfiles", new FileInputStream(f), ContentType.APPLICATION_OCTET_STREAM, f.getName()); + + HttpEntity multipart = builder.build(); + uploadFile.setEntity(multipart); + CloseableHttpResponse response = httpClient.execute(uploadFile); + HttpEntity responseEntity = response.getEntity(); + return EntityUtils.toString(responseEntity, "UTF-8"); + } + + + public static String getStatus(String url, + String userId, + String token, + String jobId) throws IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + HttpPost uploadFile = new HttpPost(url); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("jobId", jobId, ContentType.TEXT_PLAIN); + builder.addTextBody("token", token, ContentType.TEXT_PLAIN); + builder.addTextBody("userId", userId, ContentType.TEXT_PLAIN); + + HttpEntity multipart = builder.build(); + uploadFile.setEntity(multipart); + CloseableHttpResponse response = httpClient.execute(uploadFile); + HttpEntity responseEntity = response.getEntity(); + return EntityUtils.toString(responseEntity, "UTF-8"); + } + + + public static String exportLineage(ExportLineageReq req) throws IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + HttpPost uploadFile = new HttpPost(req.getUrl()); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("jobId", req.getJobId(), ContentType.TEXT_PLAIN); + builder.addTextBody("userId", req.getUserId(), ContentType.TEXT_PLAIN); + builder.addTextBody("token", req.getToken(), ContentType.TEXT_PLAIN); + builder.addTextBody("tableToTable", String.valueOf(req.getTableToTable()), ContentType.TEXT_PLAIN); + + HttpEntity multipart = builder.build(); + uploadFile.setEntity(multipart); + CloseableHttpResponse response = httpClient.execute(uploadFile); + + HttpEntity responseEntity = response.getEntity(); + + InputStream in = responseEntity.getContent(); + FileUtil.mkFile(req.getDownloadFilePath()); + File file = new File(req.getDownloadFilePath()); + FileOutputStream fout = new FileOutputStream(file); + int a; + byte[] tmp = new byte[1024]; + while ((a = in.read(tmp)) != -1) { + fout.write(tmp, 0, a); + } + fout.flush(); + fout.close(); + in.close(); + return "download success, path:" + req.getDownloadFilePath(); + } + + + private static String doPost(String url, Map param) { + CloseableHttpClient httpClient = HttpClients.createDefault(); + CloseableHttpResponse response = null; + String resultString = ""; + try { + HttpPost httpPost = new HttpPost(url); + if (param != null) { + List paramList = new ArrayList<>(); + for (String key : param.keySet()) { + paramList.add(new BasicNameValuePair(key, param.get(key))); + } + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramList, "utf-8"); + httpPost.setEntity(entity); + } + response = httpClient.execute(httpPost); + resultString = EntityUtils.toString(response.getEntity(), "utf-8"); + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + response.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return resultString; + } + + public static class ExportLineageReq { + + private String jobId; + private String userId; + private String token; + + private String url; + private String downloadFilePath; + private Boolean tableToTable = false; + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + + public Boolean getTableToTable() { + return tableToTable; + } + + public void setTableToTable(Boolean tableToTable) { + this.tableToTable = tableToTable; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getDownloadFilePath() { + return downloadFilePath; + } + + public void setDownloadFilePath(String downloadFilePath) { + this.downloadFilePath = downloadFilePath; + } + } + +} + + diff --git a/api/java/src/sources.list b/api/java/src/sources.list new file mode 100644 index 0000000..31bea69 --- /dev/null +++ b/api/java/src/sources.list @@ -0,0 +1,4 @@ +/Users/g7/Documents/project/sqlflow_public/api/java/src/main/java/com/gudusoft/grabit/FileUtil.java +/Users/g7/Documents/project/sqlflow_public/api/java/src/main/java/com/gudusoft/grabit/Runner.java +/Users/g7/Documents/project/sqlflow_public/api/java/src/main/java/com/gudusoft/grabit/SqlFlowUtil.java +/Users/g7/Documents/project/sqlflow_public/api/java/src/main/java/com/gudusoft/grabit/DateUtil.java diff --git a/api/job-types.png b/api/job-types.png new file mode 100644 index 0000000..d0d7d7e Binary files /dev/null and b/api/job-types.png differ diff --git a/api/php/Grabit.php b/api/php/Grabit.php new file mode 100644 index 0000000..f42563f --- /dev/null +++ b/api/php/Grabit.php @@ -0,0 +1,151 @@ +getToken($server, $userId, $userSecret); + echo 'get token successful.'; + echo PHP_EOL; + if (is_dir($sqlfiles)) { + if (substr($sqlfiles, -strlen(DIRECTORY_SEPARATOR)) === DIRECTORY_SEPARATOR) { + $sqlfiles = rtrim($sqlfiles, DIRECTORY_SEPARATOR); + } + + $zip = new \ZipArchive(); + $sqlfileDir = $sqlfiles . '.zip'; + if (file_exists($sqlfileDir)) { + if (PATH_SEPARATOR == ':') { + unlink($sqlfileDir); + } else { + $url = iconv('utf-8', 'gbk', $sqlfileDir); + unlink($url); + } + } + + $open = $zip->open($sqlfileDir, \ZipArchive::CREATE); + if ($open === true) { + $this->toZip($sqlfiles, $zip); + $zip->close(); + } + $sqlfiles = $sqlfileDir; + } + + echo 'start submit job.'; + echo PHP_EOL; + + $result = $obj->submitJob($server, $userId, $token, $sqlfiles, time(), $dbvendor); + if ($result['code'] == 200) { + echo 'submit job successful.'; + echo PHP_EOL; + + $jobId = $result['data']['jobId']; + while (true) { + $result = $obj->getStatus($server, $userId, $token, $jobId); + if ($result['code'] == 200) { + $status = $result['data']['status']; + if ($status == 'partial_success' || $status == 'success') { + break; + } + if ($status == 'fail') { + echo 'job execution failed.'; + exit(1); + } + } + } + echo $status; + echo 'start get result from sqlflow.'; + echo PHP_EOL; + $filePath = $obj->getResult($server, $userId, $token, $jobId, $download); + echo 'get result from sqlflow successful. file path is : ' . $filePath; + } else { + echo 'submit job failed.'; + } + echo PHP_EOL; + echo '===================================== end ====================================='; + } + + function toZip($path, $zip) + { + $handler = opendir($path); + while (($filename = readdir($handler)) !== false) { + if ($filename != "." && $filename != "..") { + if (is_dir($path . DIRECTORY_SEPARATOR . $filename)) { + $obj = new Grabit(); + $obj->toZip($path . DIRECTORY_SEPARATOR . $filename, $zip); + } else { + $zip->addFile($path . DIRECTORY_SEPARATOR . $filename); + $zip->renameName($path . DIRECTORY_SEPARATOR . $filename, $filename); + } + } + } + @closedir($path); + } +} + +$obj = new Grabit(); +$obj->run($argv); diff --git a/api/php/HttpClient.php b/api/php/HttpClient.php new file mode 100644 index 0000000..1cba8b0 --- /dev/null +++ b/api/php/HttpClient.php @@ -0,0 +1,110 @@ + $content) { + $data .= "--" . static::$delimiter . "\r\n" + . 'Content-Disposition: form-data; name="' . $name . "\"\r\n\r\n" + . $content . "\r\n"; + } + $data .= "--" . static::$delimiter . $eol + . 'Content-Disposition: form-data; name="sqlfiles"; filename="' . $param['filename'] . '"' . "\r\n" + . 'Content-Type:application/octet-stream' . "\r\n\r\n"; + + $data .= $upload . "\r\n"; + $data .= "--" . static::$delimiter . "--\r\n"; + return $data; + } + + function postFile($url, $param) + { + $post_data = static::buildData($param); + $curl = curl_init($url); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_POST, true); + curl_setopt($curl, CURLOPT_POSTFIELDS, $post_data); + curl_setopt($curl, CURLOPT_HTTPHEADER, [ + "Content-Type: multipart/form-data; boundary=" . static::$delimiter, + "Content-Length: " . strlen($post_data) + ]); + $response = curl_exec($curl); + curl_close($curl); + $info = json_decode($response, true); + return $info; + } + + + function postFrom($url, $data) + { + $headers = array('Content-Type: application/x-www-form-urlencoded'); + $curl = curl_init(); + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0); + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 1); + curl_setopt($curl, CURLOPT_AUTOREFERER, 1); + curl_setopt($curl, CURLOPT_POST, 1); + curl_setopt($curl, CURLOPT_POSTFIELDS, http_build_query($data)); + curl_setopt($curl, CURLOPT_TIMEOUT, 30); + curl_setopt($curl, CURLOPT_HEADER, 0); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); + $result = curl_exec($curl); + if (curl_errno($curl)) { + return 'Errno' . curl_error($curl); + } + curl_close($curl); + return json_decode($result, true); + } + + + function postJson($url, $data, $filePath) + { + $headers = array('Content-Type: application/x-www-form-urlencoded'); + $curl = curl_init(); + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0); + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 1); + curl_setopt($curl, CURLOPT_AUTOREFERER, 1); + curl_setopt($curl, CURLOPT_POST, 1); + curl_setopt($curl, CURLOPT_POSTFIELDS, http_build_query($data)); + curl_setopt($curl, CURLOPT_TIMEOUT, 30); + curl_setopt($curl, CURLOPT_HEADER, 0); + curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); + $result = curl_exec($curl); + if (curl_errno($curl)) { + return 'Errno' . curl_error($curl); + } + $fp = @fopen($filePath, "a"); + fwrite($fp, $result); + fclose($fp); + } + +} diff --git a/api/php/SqlFlowUtil.php b/api/php/SqlFlowUtil.php new file mode 100644 index 0000000..c03b89c --- /dev/null +++ b/api/php/SqlFlowUtil.php @@ -0,0 +1,72 @@ +postFrom($url, $json); + return $result['token']; + } + + function submitJob($server, $userId, $token, $sqlfiles, $jobName, $dbvendor) + { + $httpVendor = new HttpClient(); + $params = array( + 'userId' => $userId, + 'token' => $token, + 'jobName' => $jobName, + 'dbvendor' => $dbvendor, + 'filename' => $jobName, + 'sqlfiles' => file_get_contents($sqlfiles) + ); + $url = $server . '/gspLive_backend/sqlflow/job/submitUserJob'; + $result = $httpVendor->postFile($url, $params); + return $result; + } + + function getStatus($server, $userId, $token, $jobId) + { + $httpVendor = new HttpClient(); + $json['userId'] = $userId; + $json['token'] = $token; + $json['jobId'] = $jobId; + $url = $server . '/gspLive_backend/sqlflow/job/displayUserJobSummary'; + $result = $httpVendor->postFrom($url, $json); + return $result; + } + + function getResult($server, $userId, $token, $jobId, $download) + { + $dir = 'data' . DIRECTORY_SEPARATOR . 'result'; + $str = $dir . DIRECTORY_SEPARATOR . date("Ymd") . '_' . $jobId; + $filePath = ''; + $url = ''; + if ($download == 1) { + $url = $server . '/gspLive_backend/sqlflow/job/exportLineageAsJson'; + $filePath = $str . '_json.json'; + } else if ($download == 2) { + $url = $server . '/gspLive_backend/sqlflow/job/exportLineageAsGraphml'; + $filePath = $str . '_graphml.graphml'; + } else if ($download == 3) { + $url = $server . '/gspLive_backend/sqlflow/job/exportLineageAsCsv'; + $filePath = $str . '_csv.csv'; + } + + $httpVendor = new HttpClient(); + $json['userId'] = $userId; + $json['token'] = $token; + $json['jobId'] = $jobId; + $httpVendor->mkdirs($dir); + $httpVendor->postJson($url, $json, $filePath); + return $filePath; + } +} \ No newline at end of file diff --git a/api/php/php-data-lineage-overview.png b/api/php/php-data-lineage-overview.png new file mode 100644 index 0000000..b0c1f5b Binary files /dev/null and b/api/php/php-data-lineage-overview.png differ diff --git a/api/php/php-data-lineage-result.json b/api/php/php-data-lineage-result.json new file mode 100644 index 0000000..b5eb1f8 --- /dev/null +++ b/api/php/php-data-lineage-result.json @@ -0,0 +1,4973 @@ +{ + "code": 203, + "data": { + "mode": "global", + "summary": { + "schema": 4, + "database": 0, + "view": 2, + "mostRelationTables": [ + { + "schema": "SALES", + "table": "SALESPERSON" + }, + { + "schema": "PERSON", + "table": "PERSON" + }, + { + "table": "VIEW1" + } + ], + "column": 34, + "table": 8, + "relation": 33 + }, + "sqlflow": { + "dbvendor": "dbvmssql", + "dbobjs": [ + { + "id": "54", + "schema": "dbo", + "name": "dbo.uspGetEmployeeSales", + "type": "procedure", + "arguments": [], + "coordinates": [ + { + "x": 9, + "y": 18, + "hashCode": "0" + }, + { + "x": 9, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "8", + "schema": "HumanResources", + "name": "HumanResources.Employee", + "alias": "e", + "type": "table", + "columns": [ + { + "id": "113", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 35, + "y": 28, + "hashCode": "0" + }, + { + "x": 35, + "y": 46, + "hashCode": "0" + }, + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "11", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 34, + "y": 6, + "hashCode": "0" + }, + { + "x": 34, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "13", + "schema": "Person", + "name": "Person.Person", + "alias": "p", + "type": "table", + "columns": [ + { + "id": "118", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 35, + "y": 49, + "hashCode": "0" + }, + { + "x": 35, + "y": 67, + "hashCode": "0" + }, + { + "x": 16, + "y": 34, + "hashCode": "0" + }, + { + "x": 16, + "y": 52, + "hashCode": "0" + }, + { + "x": 25, + "y": 34, + "hashCode": "0" + }, + { + "x": 25, + "y": 52, + "hashCode": "0" + }, + { + "x": 51, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 51, + "hashCode": "0" + }, + { + "x": 53, + "y": 26, + "hashCode": "0" + }, + { + "x": 53, + "y": 44, + "hashCode": "0" + } + ] + }, + { + "id": "15", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "119", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 35, + "y": 6, + "hashCode": "0" + }, + { + "x": 35, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "30", + "name": "FactInternetSales", + "alias": "fis", + "type": "table", + "columns": [ + { + "id": "114", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 44, + "y": 5, + "hashCode": "0" + }, + { + "x": 44, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "32", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "33", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "34", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 42, + "y": 6, + "hashCode": "0" + }, + { + "x": 42, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "37", + "name": "DimSalesTerritory", + "alias": "dst", + "type": "table", + "columns": [ + { + "id": "38", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 44, + "y": 27, + "hashCode": "0" + }, + { + "x": 44, + "y": 48, + "hashCode": "0" + } + ] + }, + { + "id": "39", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 43, + "y": 17, + "hashCode": "0" + }, + { + "x": 43, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "56", + "schema": "Sales", + "name": "Sales.SalesPerson", + "alias": "sp", + "type": "table", + "columns": [ + { + "id": "115", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "116", + "name": "SalesYTD", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "111", + "name": "TerritoryID", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 14, + "y": 10, + "hashCode": "0" + }, + { + "x": 14, + "y": 33, + "hashCode": "0" + } + ] + }, + { + "id": "94", + "schema": "Person", + "name": "Person.Address", + "alias": "a", + "type": "table", + "columns": [ + { + "id": "95", + "name": "AddressID", + "coordinates": [ + { + "x": 53, + "y": 12, + "hashCode": "0" + }, + { + "x": 53, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "117", + "name": "PostalCode", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 52, + "y": 16, + "hashCode": "0" + }, + { + "x": 52, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "2", + "schema": "dbo", + "name": "dbo.EmployeeSales", + "type": "table", + "columns": [ + { + "id": "3", + "name": "DataSource", + "coordinates": [ + { + "x": 3, + "y": 3, + "hashCode": "0" + }, + { + "x": 3, + "y": 13, + "hashCode": "0" + } + ] + }, + { + "id": "4", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 4, + "y": 3, + "hashCode": "0" + }, + { + "x": 4, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "5", + "name": "LastName", + "coordinates": [ + { + "x": 5, + "y": 3, + "hashCode": "0" + }, + { + "x": 5, + "y": 11, + "hashCode": "0" + } + ] + }, + { + "id": "6", + "name": "SalesDollars", + "coordinates": [ + { + "x": 6, + "y": 3, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "87", + "name": "DUMMY0", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "1", + "name": "PseudoRows", + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 31, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "109", + "name": "pseudo_table_include_orphan_column", + "type": "pseudoTable", + "columns": [ + { + "id": "110", + "name": "TerritoryID", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 1, + "y": 1, + "hashCode": "0" + }, + { + "x": 1, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "24", + "name": "hiredate_view", + "type": "view", + "columns": [ + { + "id": "25", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "26", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "27", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "28", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "23", + "name": "PseudoRows", + "coordinates": [ + { + "x": 31, + "y": 13, + "hashCode": "0" + }, + { + "x": 31, + "y": 26, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 31, + "y": 13, + "hashCode": "0" + }, + { + "x": 31, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "48", + "name": "view1", + "type": "view", + "columns": [ + { + "id": "49", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "50", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "51", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "52", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "53", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "47", + "name": "PseudoRows", + "coordinates": [ + { + "x": 38, + "y": 13, + "hashCode": "0" + }, + { + "x": 38, + "y": 18, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 38, + "y": 13, + "hashCode": "0" + }, + { + "x": 38, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "id": "18", + "name": "RS-1", + "type": "select_list", + "columns": [ + { + "id": "19", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "20", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "21", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "22", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "17", + "name": "PseudoRows", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "41", + "name": "RS-2", + "type": "select_list", + "columns": [ + { + "id": "42", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "43", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "44", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "45", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "46", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "40", + "name": "PseudoRows", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "66", + "name": "RS-3", + "type": "select_list", + "columns": [ + { + "id": "67", + "name": "PROCEDURE", + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 12, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "69", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + } + ] + }, + { + "id": "70", + "name": "LastName", + "coordinates": [ + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + } + ] + }, + { + "id": "71", + "name": "SalesYTD", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "65", + "name": "PseudoRows", + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "81", + "name": "INSERT-SELECT-1", + "type": "insert-select", + "columns": [ + { + "id": "82", + "name": "SELECT", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "84", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "85", + "name": "LastName", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + }, + { + "id": "86", + "name": "SalesYTD", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + { + "id": "80", + "name": "PseudoRows", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + { + "id": "98", + "name": "RS-4", + "type": "select_list", + "columns": [ + { + "id": "99", + "name": "\"Row Number\"", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 47, + "y": 89, + "hashCode": "0" + } + ] + }, + { + "id": "105", + "name": "LastName", + "coordinates": [ + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "106", + "name": "SalesYTD", + "coordinates": [ + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + } + ] + }, + { + "id": "107", + "name": "PostalCode", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "97", + "name": "PseudoRows", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + } + ], + "relations": [ + { + "id": "4101", + "type": "fdd", + "effectType": "select", + "target": { + "id": "107", + "column": "PostalCode", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "117", + "column": "PostalCode", + "parentId": "94", + "parentName": "Person.Address", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4102", + "type": "fdd", + "effectType": "select", + "target": { + "id": "69", + "column": "BusinessEntityID", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "115", + "column": "BusinessEntityID", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4103", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "52", + "column": "SalesTerritoryKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "45", + "column": "SalesTerritoryKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4104", + "type": "fdd", + "effectType": "select", + "target": { + "id": "21", + "column": "BusinessEntityID", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "113", + "column": "BusinessEntityID", + "parentId": "8", + "parentName": "HumanResources.Employee", + "coordinates": [ + { + "x": 35, + "y": 28, + "hashCode": "0" + }, + { + "x": 35, + "y": 46, + "hashCode": "0" + }, + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4105", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "26", + "column": "LastName", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "20", + "column": "LastName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4106", + "type": "fdd", + "effectType": "select", + "target": { + "id": "20", + "column": "LastName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4107", + "type": "fdd", + "effectType": "select", + "target": { + "id": "99", + "column": "\"Row Number\"", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 47, + "y": 89, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "117", + "column": "PostalCode", + "parentId": "94", + "parentName": "Person.Address", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4108", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "28", + "column": "HireDate", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "22", + "column": "HireDate", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4109", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "27", + "column": "BusinessEntityID", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "21", + "column": "BusinessEntityID", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4110", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22", + "column": "HireDate", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "11", + "column": "HireDate", + "parentId": "8", + "parentName": "HumanResources.Employee", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4111", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "4", + "column": "BusinessEntityID", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 4, + "y": 3, + "hashCode": "0" + }, + { + "x": 4, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "84", + "column": "BusinessEntityID", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4112", + "type": "fdd", + "effectType": "select", + "target": { + "id": "106", + "column": "SalesYTD", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4113", + "type": "fdd", + "effectType": "select", + "target": { + "id": "86", + "column": "SalesYTD", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4114", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "51", + "column": "OrderDateKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "44", + "column": "OrderDateKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4115", + "type": "fdd", + "effectType": "select", + "target": { + "id": "70", + "column": "LastName", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4116", + "type": "fdd", + "effectType": "select", + "target": { + "id": "71", + "column": "SalesYTD", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4117", + "type": "fdd", + "effectType": "select", + "target": { + "id": "43", + "column": "ProductKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "33", + "column": "ProductKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4118", + "type": "fdd", + "effectType": "select", + "target": { + "id": "85", + "column": "LastName", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4119", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "50", + "column": "ProductKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "43", + "column": "ProductKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4120", + "type": "fdd", + "effectType": "select", + "target": { + "id": "44", + "column": "OrderDateKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "34", + "column": "OrderDateKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4121", + "type": "fdd", + "effectType": "select", + "target": { + "id": "42", + "column": "CustomerKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "32", + "column": "CustomerKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4122", + "type": "fdd", + "effectType": "select", + "target": { + "id": "84", + "column": "BusinessEntityID", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "115", + "column": "BusinessEntityID", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4123", + "type": "fdd", + "effectType": "select", + "target": { + "id": "19", + "column": "FirstName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "15", + "column": "FirstName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4124", + "type": "fdd", + "effectType": "select", + "target": { + "id": "105", + "column": "LastName", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4125", + "type": "fdd", + "effectType": "select", + "target": { + "id": "45", + "column": "SalesTerritoryKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "114", + "column": "SalesTerritoryKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 44, + "y": 5, + "hashCode": "0" + }, + { + "x": 44, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4126", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "49", + "column": "CustomerKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "42", + "column": "CustomerKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4127", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "5", + "column": "LastName", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 5, + "y": 3, + "hashCode": "0" + }, + { + "x": 5, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "85", + "column": "LastName", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4128", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "6", + "column": "SalesDollars", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 6, + "y": 3, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "86", + "column": "SalesYTD", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4129", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "25", + "column": "FirstName", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "19", + "column": "FirstName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4130", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "87", + "column": "DUMMY0", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "82", + "column": "SELECT", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4131", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "53", + "column": "SalesTerritoryRegion", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "46", + "column": "SalesTerritoryRegion", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4132", + "type": "fdd", + "effectType": "select", + "target": { + "id": "46", + "column": "SalesTerritoryRegion", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "39", + "column": "SalesTerritoryRegion", + "parentId": "37", + "parentName": "DimSalesTerritory", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ] + } + ], + "errors": [ + { + "errorMessage": "find orphan column(10500) near: PostalCode(47,39)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: SalesYTD(47,59)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: TerritoryID(54,7)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: SalesYTD(55,9)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: PostalCode(56,10)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 56, + "y": 10, + "hashCode": "0" + }, + { + "x": 56, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "Link orphan column [TerritoryID] to the first table [Sales.SalesPerson s]", + "errorType": "LinkOrphanColumn", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ] + }, + "graph": { + "elements": { + "tables": [ + { + "columns": [ + { + "height": 16, + "id": "n0::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -625.07874 + }, + { + "height": 16, + "id": "n0::n1", + "label": { + "content": "HIREDATE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 76, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -609.07874 + } + ], + "height": 57.96875, + "id": "n0", + "label": { + "content": "HumanResources.Employee", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 178, + "x": -8, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -647.0475 + }, + { + "columns": [ + { + "height": 16, + "id": "n1::n0", + "label": { + "content": "FIRSTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 84, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -537.1097 + }, + { + "height": 16, + "id": "n1::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -521.1097 + } + ], + "height": 57.96875, + "id": "n1", + "label": { + "content": "Person.Person", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -559.0784 + }, + { + "columns": [ + { + "height": 16, + "id": "n2::n0", + "label": { + "content": "SALESTERRITORYKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 141, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -78.00015 + }, + { + "height": 16, + "id": "n2::n1", + "label": { + "content": "CUSTOMERKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 105, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -62.00015 + }, + { + "height": 16, + "id": "n2::n2", + "label": { + "content": "PRODUCTKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -46.00015 + }, + { + "height": 16, + "id": "n2::n3", + "label": { + "content": "ORDERDATEKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 112, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -30.000149 + } + ], + "height": 89.96875, + "id": "n2", + "label": { + "content": "FactInternetSales", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -99.9689 + }, + { + "columns": [ + { + "height": 16, + "id": "n3::n0", + "label": { + "content": "SALESTERRITORYREGION", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 165, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -149.9692 + } + ], + "height": 41.96875, + "id": "n3", + "label": { + "content": "DimSalesTerritory", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -171.93794 + }, + { + "columns": [ + { + "height": 16, + "id": "n4::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -362.57196 + }, + { + "height": 16, + "id": "n4::n1", + "label": { + "content": "SALESYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 77, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -346.57196 + } + ], + "height": 57.96875, + "id": "n4", + "label": { + "content": "Sales.SalesPerson", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -384.5407 + }, + { + "columns": [ + { + "height": 16, + "id": "n5::n0", + "label": { + "content": "POSTALCODE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -221.93825 + } + ], + "height": 41.96875, + "id": "n5", + "label": { + "content": "Person.Address", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -243.907 + }, + { + "columns": [ + { + "height": 16, + "id": "n6::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -495.37158 + }, + { + "height": 16, + "id": "n6::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -479.37158 + }, + { + "height": 16, + "id": "n6::n2", + "label": { + "content": "SALESDOLLARS", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 108, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -463.37158 + }, + { + "height": 16, + "id": "n6::n3", + "label": { + "content": "DUMMY0", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 69, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -447.37158 + } + ], + "height": 89.96875, + "id": "n6", + "label": { + "content": "dbo.EmployeeSales", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -517.34033 + }, + { + "columns": [ + { + "height": 16, + "id": "n7::n0", + "label": { + "content": "FIRSTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 84, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -615.34064 + }, + { + "height": 16, + "id": "n7::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -599.34064 + }, + { + "height": 16, + "id": "n7::n2", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -583.34064 + }, + { + "height": 16, + "id": "n7::n3", + "label": { + "content": "HIREDATE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 76, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -567.34064 + } + ], + "height": 89.96875, + "id": "n7", + "label": { + "content": "hiredate_view", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -637.3094 + }, + { + "columns": [ + { + "height": 16, + "id": "n8::n0", + "label": { + "content": "CUSTOMERKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 105, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -97.24618 + }, + { + "height": 16, + "id": "n8::n1", + "label": { + "content": "PRODUCTKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -81.24618 + }, + { + "height": 16, + "id": "n8::n2", + "label": { + "content": "ORDERDATEKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 112, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -65.24618 + }, + { + "height": 16, + "id": "n8::n3", + "label": { + "content": "SALESTERRITORYKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 141, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -49.24618 + }, + { + "height": 16, + "id": "n8::n4", + "label": { + "content": "SALESTERRITORYREGION", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 165, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -33.24618 + } + ], + "height": 105.96875, + "id": "n8", + "label": { + "content": "view1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -119.21493 + }, + { + "columns": [ + { + "height": 16, + "id": "n9::n0", + "label": { + "content": "FirstName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 79, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -615.34064 + }, + { + "height": 16, + "id": "n9::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -599.34064 + }, + { + "height": 16, + "id": "n9::n2", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -583.34064 + }, + { + "height": 16, + "id": "n9::n3", + "label": { + "content": "HireDate", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 71, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -567.34064 + } + ], + "height": 89.96875, + "id": "n9", + "label": { + "content": "RS-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -637.3094 + }, + { + "columns": [ + { + "height": 16, + "id": "n10::n0", + "label": { + "content": "CustomerKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 97, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -97.24618 + }, + { + "height": 16, + "id": "n10::n1", + "label": { + "content": "ProductKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 86, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -81.24618 + }, + { + "height": 16, + "id": "n10::n2", + "label": { + "content": "OrderDateKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 102, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -65.24618 + }, + { + "height": 16, + "id": "n10::n3", + "label": { + "content": "SalesTerritoryKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 123, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -49.24618 + }, + { + "height": 16, + "id": "n10::n4", + "label": { + "content": "SalesTerritoryRegion", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 145, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -33.24618 + } + ], + "height": 105.96875, + "id": "n10", + "label": { + "content": "RS-2", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -119.21493 + }, + { + "columns": [ + { + "height": 16, + "id": "n11::n0", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -375.40253 + }, + { + "height": 16, + "id": "n11::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -359.40253 + }, + { + "height": 16, + "id": "n11::n2", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -343.40253 + } + ], + "height": 73.96875, + "id": "n11", + "label": { + "content": "RS-3", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -397.37128 + }, + { + "columns": [ + { + "height": 16, + "id": "n12::n0", + "label": { + "content": "SELECT", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -495.37158 + }, + { + "height": 16, + "id": "n12::n1", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -479.37158 + }, + { + "height": 16, + "id": "n12::n2", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -463.37158 + }, + { + "height": 16, + "id": "n12::n3", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -447.37158 + } + ], + "height": 89.96875, + "id": "n12", + "label": { + "content": "INSERT-SELECT-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -517.34033 + }, + { + "columns": [ + { + "height": 16, + "id": "n13::n0", + "label": { + "content": "\"Row Number\"", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 104, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -271.43347 + }, + { + "height": 16, + "id": "n13::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -255.43349 + }, + { + "height": 16, + "id": "n13::n2", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -239.43349 + }, + { + "height": 16, + "id": "n13::n3", + "label": { + "content": "PostalCode", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 87, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -223.43349 + } + ], + "height": 89.96875, + "id": "n13", + "label": { + "content": "RS-4", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -293.40222 + } + ], + "edges": [ + { + "id": "e0", + "sourceId": "n4::n0", + "targetId": "n12::n1" + }, + { + "id": "e1", + "sourceId": "n9::n1", + "targetId": "n7::n1" + }, + { + "id": "e2", + "sourceId": "n2::n3", + "targetId": "n10::n2" + }, + { + "id": "e3", + "sourceId": "n9::n3", + "targetId": "n7::n3" + }, + { + "id": "e4", + "sourceId": "n1::n1", + "targetId": "n13::n1" + }, + { + "id": "e5", + "sourceId": "n1::n1", + "targetId": "n12::n2" + }, + { + "id": "e6", + "sourceId": "n10::n4", + "targetId": "n8::n4" + }, + { + "id": "e7", + "sourceId": "n5::n0", + "targetId": "n13::n3" + }, + { + "id": "e8", + "sourceId": "n2::n2", + "targetId": "n10::n1" + }, + { + "id": "e9", + "sourceId": "n1::n1", + "targetId": "n11::n1" + }, + { + "id": "e10", + "sourceId": "n12::n3", + "targetId": "n6::n2" + }, + { + "id": "e11", + "sourceId": "n2::n1", + "targetId": "n10::n0" + }, + { + "id": "e12", + "sourceId": "n4::n1", + "targetId": "n11::n2" + }, + { + "id": "e13", + "sourceId": "n12::n2", + "targetId": "n6::n1" + }, + { + "id": "e14", + "sourceId": "n1::n1", + "targetId": "n9::n1" + }, + { + "id": "e15", + "sourceId": "n12::n1", + "targetId": "n6::n0" + }, + { + "id": "e16", + "sourceId": "n1::n0", + "targetId": "n9::n0" + }, + { + "id": "e17", + "sourceId": "n9::n2", + "targetId": "n7::n2" + }, + { + "id": "e18", + "sourceId": "n4::n1", + "targetId": "n12::n3" + }, + { + "id": "e19", + "sourceId": "n0::n1", + "targetId": "n9::n3" + }, + { + "id": "e20", + "sourceId": "n9::n0", + "targetId": "n7::n0" + }, + { + "id": "e21", + "sourceId": "n3::n0", + "targetId": "n10::n4" + }, + { + "id": "e22", + "sourceId": "n12::n0", + "targetId": "n6::n3" + }, + { + "id": "e23", + "sourceId": "n2::n0", + "targetId": "n10::n3" + }, + { + "id": "e24", + "sourceId": "n0::n0", + "targetId": "n9::n2" + }, + { + "id": "e25", + "sourceId": "n10::n2", + "targetId": "n8::n2" + }, + { + "id": "e26", + "sourceId": "n4::n1", + "targetId": "n13::n2" + }, + { + "id": "e27", + "sourceId": "n4::n1", + "targetId": "n13::n0" + }, + { + "id": "e28", + "sourceId": "n5::n0", + "targetId": "n13::n0" + }, + { + "id": "e29", + "sourceId": "n10::n1", + "targetId": "n8::n1" + }, + { + "id": "e30", + "sourceId": "n10::n0", + "targetId": "n8::n0" + }, + { + "id": "e31", + "sourceId": "n10::n3", + "targetId": "n8::n3" + }, + { + "id": "e32", + "sourceId": "n4::n0", + "targetId": "n11::n0" + } + ] + }, + "tooltip": { + "pseudo_tab...rphan_column": "pseudo_table_include_orphan_column" + }, + "relationIdMap": { + "e0": "fdd", + "e1": "fdd", + "e2": "fdd", + "e3": "fdd", + "e4": "fdd", + "e5": "fdd", + "e6": "fdd", + "e7": "fdd", + "e8": "fdd", + "e9": "fdd", + "e10": "fdd", + "e11": "fdd", + "e12": "fdd", + "e13": "fdd", + "e14": "fdd", + "e15": "fdd", + "e16": "fdd", + "e17": "fdd", + "e18": "fdd", + "e19": "fdd", + "e20": "fdd", + "e21": "fdd", + "e22": "fdd", + "e23": "fdd", + "e24": "fdd", + "e25": "fdd", + "e26": "fdd", + "e27": "fdd", + "e28": "fdd", + "e29": "fdd", + "e30": "fdd", + "e31": "fdd", + "e32": "fdd" + }, + "listIdMap": { + "n0": [ + "8" + ], + "n0::n0": [ + "113" + ], + "n0::n1": [ + "11" + ], + "n1": [ + "13" + ], + "n1::n0": [ + "15" + ], + "n1::n1": [ + "119" + ], + "n2": [ + "30" + ], + "n2::n0": [ + "114" + ], + "n2::n1": [ + "32" + ], + "n2::n2": [ + "33" + ], + "n2::n3": [ + "34" + ], + "n3": [ + "37" + ], + "n3::n0": [ + "39" + ], + "n4": [ + "56" + ], + "n4::n0": [ + "115" + ], + "n4::n1": [ + "116" + ], + "n5": [ + "94" + ], + "n5::n0": [ + "117" + ], + "n6": [ + "2" + ], + "n6::n0": [ + "4" + ], + "n6::n1": [ + "5" + ], + "n6::n2": [ + "6" + ], + "n6::n3": [ + "87" + ], + "n7": [ + "24" + ], + "n7::n0": [ + "25" + ], + "n7::n1": [ + "26" + ], + "n7::n2": [ + "27" + ], + "n7::n3": [ + "28" + ], + "n8": [ + "48" + ], + "n8::n0": [ + "49" + ], + "n8::n1": [ + "50" + ], + "n8::n2": [ + "51" + ], + "n8::n3": [ + "52" + ], + "n8::n4": [ + "53" + ], + "n9": [ + "18" + ], + "n9::n0": [ + "19" + ], + "n9::n1": [ + "20" + ], + "n9::n2": [ + "21" + ], + "n9::n3": [ + "22" + ], + "n10": [ + "41" + ], + "n10::n0": [ + "42" + ], + "n10::n1": [ + "43" + ], + "n10::n2": [ + "44" + ], + "n10::n3": [ + "45" + ], + "n10::n4": [ + "46" + ], + "n11": [ + "66" + ], + "n11::n0": [ + "69" + ], + "n11::n1": [ + "70" + ], + "n11::n2": [ + "71" + ], + "n12": [ + "81" + ], + "n12::n0": [ + "82" + ], + "n12::n1": [ + "84" + ], + "n12::n2": [ + "85" + ], + "n12::n3": [ + "86" + ], + "n13": [ + "98" + ], + "n13::n0": [ + "99" + ], + "n13::n1": [ + "105" + ], + "n13::n2": [ + "106" + ], + "n13::n3": [ + "107" + ] + } + } + }, + "error": "find orphan column(10500) near: PostalCode(47,39)\r\nfind orphan column(10500) near: SalesYTD(47,59)\r\nfind orphan column(10500) near: TerritoryID(54,7)\r\nfind orphan column(10500) near: SalesYTD(55,9)\r\nfind orphan column(10500) near: PostalCode(56,10)\r\nLink orphan column [TerritoryID] to the first table [Sales.SalesPerson s]", + "sessionId": "ec3938ae68a8a468d729e6c426108e701e78079cf070a385acf7cd899d41058c_1615189437331" +} \ No newline at end of file diff --git a/api/php/php-data-lineage-sqlserver.sql b/api/php/php-data-lineage-sqlserver.sql new file mode 100644 index 0000000..8a6c74f --- /dev/null +++ b/api/php/php-data-lineage-sqlserver.sql @@ -0,0 +1,56 @@ +-- sql server sample sql +CREATE TABLE dbo.EmployeeSales +( DataSource varchar(20) NOT NULL, + BusinessEntityID varchar(11) NOT NULL, + LastName varchar(40) NOT NULL, + SalesDollars money NOT NULL +); +GO +CREATE PROCEDURE dbo.uspGetEmployeeSales +AS + SET NOCOUNT ON; + SELECT 'PROCEDURE', sp.BusinessEntityID, c.LastName, + sp.SalesYTD + FROM Sales.SalesPerson AS sp + INNER JOIN Person.Person AS c + ON sp.BusinessEntityID = c.BusinessEntityID + WHERE sp.BusinessEntityID LIKE '2%' + ORDER BY sp.BusinessEntityID, c.LastName; +GO +--INSERT...SELECT example +INSERT INTO dbo.EmployeeSales + SELECT 'SELECT', sp.BusinessEntityID, c.LastName, sp.SalesYTD + FROM Sales.SalesPerson AS sp + INNER JOIN Person.Person AS c + ON sp.BusinessEntityID = c.BusinessEntityID + WHERE sp.BusinessEntityID LIKE '2%' + ORDER BY sp.BusinessEntityID, c.LastName; +GO + + +CREATE VIEW hiredate_view +AS +SELECT p.FirstName, p.LastName, e.BusinessEntityID, e.HireDate +FROM HumanResources.Employee e +JOIN Person.Person AS p ON e.BusinessEntityID = p.BusinessEntityID ; +GO + +CREATE VIEW view1 +AS +SELECT fis.CustomerKey, fis.ProductKey, fis.OrderDateKey, + fis.SalesTerritoryKey, dst.SalesTerritoryRegion +FROM FactInternetSales AS fis +LEFT OUTER JOIN DimSalesTerritory AS dst +ON (fis.SalesTerritoryKey=dst.SalesTerritoryKey); + +GO +SELECT ROW_NUMBER() OVER(PARTITION BY PostalCode ORDER BY SalesYTD DESC) AS "Row Number", + p.LastName, s.SalesYTD, a.PostalCode +FROM Sales.SalesPerson AS s + INNER JOIN Person.Person AS p + ON s.BusinessEntityID = p.BusinessEntityID + INNER JOIN Person.Address AS a + ON a.AddressID = p.BusinessEntityID +WHERE TerritoryID IS NOT NULL + AND SalesYTD <> 0 +ORDER BY PostalCode; \ No newline at end of file diff --git a/api/php/php-data-lineage.png b/api/php/php-data-lineage.png new file mode 100644 index 0000000..14955be Binary files /dev/null and b/api/php/php-data-lineage.png differ diff --git a/api/php/readme.md b/api/php/readme.md new file mode 100644 index 0000000..74a1865 --- /dev/null +++ b/api/php/readme.md @@ -0,0 +1,192 @@ +## PHP Data lineage: using the SQLFlow REST API (Advanced) + +This article illustrates how to discover the data lineage using PHP and the SQLFlow REST API. + +By using the SQLFlow REST API, you can code in PHP to discover the data lineage in SQL scripts +and get the result in an actionable diagram, json, csv or graphml format. + +You can integerate the PHP code provided here into your own project and add the powerful +data lineage analsysis capability instantly. + +### 1. interactive data lineage visualizations +![PHP Data lineage](php-data-lineage.png) + +### 2. [Data lineage in JSON format](php-data-lineage-result.json) + +### 3. Data lineage in CSV, graphml format + + +## Prerequisites +- [SQLFlow Cloud Server or on-premise version](https://github.com/sqlparser/sqlflow_public/tree/master/api#prerequisites) + +- PHP 7.3 or higher version must be installed and configured correctly. + +- Install the ZIP extension + +**mac** + +```` +wget http://pecl.php.net/get/zip-1.12.4.tgz + +tar zxfv zip-1.12.4.tgz + +cd zip-1.12.4 + +sudo mount -uw / + +sudo ln -s /Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk/usr/include/ /usr + +sudo phpize + +which php-config(get path,eg :/usr/bin/php-config) + +./configure --with-php-config=/usr/bin/php-config + +sudo mount -uw / + +sudo make + +sudo make install + +cd /usr/lib/php/extensions/no-debug-non-zts-20180731 + +sudo cp /private/etc/php.ini.default php.ini + +chmod 777 php.ini + +sudo vim php.ini, write extension=zip.so + +sudo apachectl restart +```` + +**linux** + +```` +wget http://pecl.php.net/get/zip-1.12.4.tgz + +tar zxfv zip-1.12.4.tgz + +cd zip-1.12.4 + +sudo phpize + +which php-config(get path,eg :/usr/bin/php-config) + +./configure --with-php-config=/usr/bin/php-config + +sudo make + +sudo make install + +cd /usr/lib/php/extensions/no-debug-non-zts-20180731 + +sudo vi /usr/local/php/etc/php.ini, write extension=zip.so + +sudo apachectl restart +```` + +#### [Reference Documentation](https://www.php.net/manual/en/install.pecl.phpize.php) + +### Usage + +```` +php Grabit.php /s server /p port /u userId /k userSecret /t databaseType /f path_to_config_file /r resultType + +eg: + php Grabit.php /u 'auth0|xxx' /k cab9712c45189014a94a8b7aceeef7a3db504be58e18cd3686f3bbefd078ef4d /s https://api.gudusoft.com /t oracle /f demo.sql /r 1 + +note: + If the parameter string contains symbols like "|" , it must be included in a single quotes (' ') +```` + +Example: + +1. Connect to the SQLFlow Cloud Server +``` +php Grabit.php /s https://api.gudusoft.com /u 'YOUR_USER_ID' /k YOUR_SECRET_KEY /t sqlserver /f PHP-data-lineage-sqlserver.sql /r 1 +``` + +2. Connect to the SQLFlow on-premise +This will discover data lineage by analyzing the `PHP-data-lineage-sqlserver.sql` file. You may also specify a zip file which includes lots of SQL files. +``` +php Grabit.php /s http://127.0.0.1 /p 8081 /u 'gudu|0123456789' /t sqlserver /f PHP-data-lineage-sqlserver.sql /r 1 +``` + +This will discover data lineage by analyzing all SQL files under `sqlfiles` directory. +``` +php Grabit.php /s http://127.0.0.1 /p 8081 /u 'gudu|0123456789' /t mysql /f sqlfiles /r 1 +``` + +### Parameters + +- **path_to_config_file** + +This can be a single SQL file, a zip file including multiple SQL files, or a directory including lots of SQL files. + +- **server** + +Usually, it is the IP address of [the SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +installed on your owner servers such as `127.0.0.1` or `http://127.0.0.1` + +You may set the value to `https://api.gudusoft.com` if you like to send your SQL script to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com) to get the data lineage result. + +- **port** + +The default value is `8081` if you connect to your SQLFlow on-premise server. + +However, if you setup the nginx reverse proxy in the nginx configuration file like this: +``` + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } +``` +Then, keep the value of `serverPort` empty and set `server` to the value like this: `http://127.0.0.1/api`. + +>Please keep this value empty if you connect to the SQLFlow Cloud Server by specifying the `https://api.gudusoft.com` +in the `server` + > +- **userId, userSecret** + +This is the user id that is used to connect to the SQLFlow server. +Always set this value to `gudu|0123456789` and keep `userSecret` empty if you use the SQLFlow on-premise version. + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + + +- **databaseType** + +This parameter specifies the database dialect of the SQL scripts that the SQLFlow has analyzed. + +```txt + access,bigquery,couchbase,dax,db2,greenplum,hana,hive,impala,informix,mdx,mssql, + sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake, + sybase,teradata,soql,vertica +``` + +- **resultType** + +When you submit SQL script to the SQLFlow server, A job is created on the SQLFlow server +and you can always see the graphic data lineage result via the browser, + + +Even better, This demo will fetch the data lineage back to the directory where the demo is running. +Those data lineage results are stored in the `data/result/` directory. + +This parameter specifies which kind of format is used to save the data lineage result. + +Available values for this parameter: +- 1: JSON, data lineage result in JSON. +- 2: CSV, data lineage result in CSV format. +- 3: diagram, in graphml format that can be viewed by yEd. + +### SQLFlow REST API +Please check here for the detailed information about the [SQLFlow REST API](https://github.com/sqlparser/sqlflow_public/tree/master/api/sqlflow_api.md) diff --git a/api/python/DataLineageParser.py b/api/python/DataLineageParser.py new file mode 100644 index 0000000..2f8fd0f --- /dev/null +++ b/api/python/DataLineageParser.py @@ -0,0 +1,77 @@ +/** +* 解析SQLFLow exportLineageAsJson接口返回的JSON格式的血缘关系中的关系链路 +* +* 例如demo中的血缘数据，解析成以下链路： +* 达成的目标是，List中两个元素： +* SCOTT.DEPT -> SCOTT.EMP->VSAL +* SCOTT.EMP->VSAL +*/ + +import json + +class Node: + def __init__(self, value, node_id): + self.value = value + self.id = node_id + self.next = None + + def key(self): + node = self.next + key = self.id + while node: + key += node.id + node = node.next + return key + +def main(): + input_data = '{"jobId":"d9550e491c024d0cbe6e1034604aca17","code":200,"data":{"mode":"global","sqlflow":{"relationship":[{"sources":[{"parentName":"ORDERS","column":"TABLE","coordinates":[],"id":"10000106","parentId":"86"}],"id":"1000012311","type":"fdd","target":{"parentName":"SPECIAL_ORDERS","column":"TABLE","coordinates":[],"id":"10000102","parentId":"82"}},{"sources":[{"parentName":"CUSTOMERS","column":"TABLE","coordinates":[],"id":"10000103","parentId":"94"}],"id":"1000012312","type":"fdd","target":{"parentName":"SPECIAL_ORDERS","column":"TABLE","coordinates":[],"id":"10000102","parentId":"82"}}]}},"sessionId":"8bb7d3da4b687bb7badf01608a739fbebd61309cd5a643cecf079d122095738a_1685604216451"}' + try: + data = json.loads(input_data) + relationship_node = data["data"]["sqlflow"]["relationships"] + data_list = relationship_node + + value = [] + node_map = {} + for data_item in data_list: + sources = data_item["sources"] + target_node = data_item["target"] + target = Node(target_node["parentName"], target_node["parentId"]) + if sources: + for source in sources: + parent_id = source["parentId"] + parent_name = source["parentName"] + source_node = Node(parent_name, parent_id) + source_node.next = target + value.append(source_node) + node_map[parent_id] = source_node + else: + value.append(target) + node_map[target_node["parentId"]] = target + + for node in value: + next_node = node.next + if next_node: + next_id = next_node.id + next_node = node_map.get(next_id) + if next_node: + node.next = next_node + + key_set = set() + value_iter = iter(value) + while True: + try: + node = next(value_iter) + k = node.key() + if k in key_set: + value_iter.remove() + key_set.add(k) + except StopIteration: + break + + chains = [] + print(chains) + except json.JSONDecodeError as e: + print(e) + +if __name__ == "__main__": + main() diff --git a/api/python/advanced/GetGenerateToken.py b/api/python/advanced/GetGenerateToken.py new file mode 100644 index 0000000..a295d0e --- /dev/null +++ b/api/python/advanced/GetGenerateToken.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import requests + +import json + + +def getToken(sys, userId, server, port): + if len(sys.argv) < 1: + print('Please enter the args.') + sys.exit(0) + + url = '/gspLive_backend/user/generateToken' + screctKey = '' + for i in range(1, len(sys.argv)): + if sys.argv[i] == '/k': + try: + if sys.argv[i + 1] is not None: + screctKey = sys.argv[i + 1] + except Exception: + print( + 'Please enter the screctKey，the secret key of sqlflow user for webapi request, required true. eg: /k xxx') + sys.exit(0) + + if port != '': + url = server + ':' + port + url + else: + url = server + url + mapA = {'secretKey': screctKey, 'userId': userId} + header_dict = {"Content-Type": "application/x-www-form-urlencoded"} + + print('start get token.') + try: + r = requests.post(url, data=mapA, headers=header_dict) + except Exception: + print('get token failed.') + sys.exit(0) + result = json.loads(r.text) + + if result['code'] == '200': + print('get token successful.') + return result['token'] + else: + print(result['error']) + sys.exit(0) diff --git a/api/python/advanced/GetJobStatus.py b/api/python/advanced/GetJobStatus.py new file mode 100644 index 0000000..35cb675 --- /dev/null +++ b/api/python/advanced/GetJobStatus.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import requests + +import json +import sys + + +def getStatus(userId, token, server, port, jobId): + url = "/gspLive_backend/sqlflow/job/displayUserJobSummary" + + if port != '': + url = server + ':' + port + url + else: + url = server + url + + data = {'jobId': jobId, 'token': token, 'userId': userId} + datastr = json.dumps(data) + + try: + response = requests.post(url, data=eval(datastr)) + except Exception: + print('get job status to sqlflow failed.') + sys.exit(0) + + result = json.loads(response.text) + if result['code'] == 200: + status = result['data']['status'] + if status == 'fail': + print(result['data']['errorMessage']) + sys.exit(0) + return status diff --git a/api/python/advanced/GetResultToSqlflow.py b/api/python/advanced/GetResultToSqlflow.py new file mode 100644 index 0000000..1fb39fb --- /dev/null +++ b/api/python/advanced/GetResultToSqlflow.py @@ -0,0 +1,51 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import requests + +import json +import sys +import os + + +def getResult(download, userId, token, server, port, jobId, filePath): + sep = 'data' + os.sep + 'result' + os.sep + filePath = filePath + '_' + jobId + if download == 'json': + url = "/gspLive_backend/sqlflow/job/exportLineageAsJson" + filePath = sep + filePath + '_json.json' + elif download == 'graphml': + url = "/gspLive_backend/sqlflow/job/exportLineageAsGraphml" + filePath = sep + filePath + '_graphml.graphml' + elif download == 'csv': + url = "/gspLive_backend/sqlflow/job/exportLineageAsCsv" + filePath = sep + filePath + '_csv.csv' + else: + print('Please enter the correct output type.') + sys.exit(0) + + if port != '': + url = server + ':' + port + url + else: + url = server + url + + data = {'jobId': jobId, 'token': token, 'userId': userId, 'tableToTable': 'false'} + datastr = json.dumps(data) + + print('start download result to sqlflow.') + try: + response = requests.post(url, data=eval(datastr)) + except Exception: + print('download result to sqlflow failed.') + sys.exit(0) + + if not os.path.exists(sep): + os.makedirs(sep) + + try: + with open(filePath, 'wb') as f: + f.write(response.content) + except Exception: + print(filePath, 'is not exist.') + sys.exit(0) + + print('download result to sqlflow successful.file path is ', filePath) diff --git a/api/python/advanced/Grabit.py b/api/python/advanced/Grabit.py new file mode 100644 index 0000000..fd6f955 --- /dev/null +++ b/api/python/advanced/Grabit.py @@ -0,0 +1,130 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import os +import sys +import GetGenerateToken +import SubmitJob +import time +import GetResultToSqlflow +import GetJobStatus +import datetime + +if __name__ == '__main__': + + print('========================================grabit-python======================================') + + userId = '' + dbvendor = '' + sqlfiles = '' + server = '' + port = '' + download = '' + + for i in range(1, len(sys.argv)): + if sys.argv[i] == '/u': + try: + if sys.argv[i + 1] is not None: + userId = sys.argv[i + 1] + else: + print( + 'Please enter the userId，the user id of sqlflow web or client, required true. eg: /n gudu|123456789') + sys.exit(0) + except BrokenPipeError: + print( + 'Please enter the userId，the user id of sqlflow web or client, required true. eg: /n gudu|123456789') + if sys.argv[i] == '/t': + try: + if sys.argv[i + 1] is not None: + dbvendor = sys.argv[i + 1] + else: + print( + 'Please enter the dbvendor.') + sys.exit(0) + except Exception: + print( + 'Please enter the dbvendor.') + if sys.argv[i] == '/f': + try: + if sys.argv[i + 1] is not None: + sqlfiles = sys.argv[i + 1] + else: + print( + 'Please enter the sqlfiles，request sql files, please use multiple parts to submit the sql files, required true. eg: /f path') + sys.exit(0) + except Exception: + print( + 'Please enter the sqlfiles，request sql files, please use multiple parts to submit the sql files, required true. eg: /f path') + if sys.argv[i] == '/s': + try: + if sys.argv[i + 1] is not None: + server = sys.argv[i + 1] + else: + print('Please enter the server. eg: /s https://api.gudusoft.com or /s https://127.0.0.1') + sys.exit(0) + except Exception: + print('Please enter the server. eg: /s https://api.gudusoft.com or /s https://127.0.0.1') + sys.exit(0) + if sys.argv[i] == '/p': + try: + if sys.argv[i + 1] is not None: + port = sys.argv[i + 1] + except Exception: + print('Please enter the port. eg: /p 8081') + sys.exit(0) + if sys.argv[i] == '/r': + try: + if sys.argv[i + 1] is not None: + download = sys.argv[i + 1] + except Exception: + print('Please enter the download type to sqlflow,type 1:json 2:csv 3:diagram : eg: /r 1') + sys.exit(0) + + if userId == '': + print('Please enter the userId，the user id of sqlflow web or client, required true. eg: /n gudu|123456789') + sys.exit(0) + if dbvendor == '': + print( + 'Please enter the dbvendor，available values:bigquery,couchbase,db2,greenplum,hana,hive,impala,informix,mdx,mysql,netezza,openedge,oracle,postgresql,redshift,snowflake,mssql,sybase,teradata,vertica. eg: /t oracle') + sys.exit(0) + + if dbvendor == 'mssql' or dbvendor == 'sqlserver': + dbvendor = 'mssql' + + dbvendor = 'dbv' + dbvendor + + if sqlfiles == '': + print( + 'Please enter the sqlfiles，request sql files, please use multiple parts to submit the sql files, required true. eg: /f path') + sys.exit(0) + if server == '': + print('Please enter the server. eg: /s https://api.gudusoft.com or /s https://127.0.0.1') + sys.exit(0) + + if server.find('http:') == -1 and server.find('https:') == -1: + server = 'http://' + server + + if server.endswith(os.sep): + server = server[:-1] + + if server == 'https://sqlflow.gudusoft.com': + server = 'https://api.gudusoft.com' + + if userId == 'gudu|0123456789': + token = 'token' + else: + token = GetGenerateToken.getToken(sys, userId, server, port) + + time_ = datetime.datetime.now().strftime('%Y%m%d') + + jobId = SubmitJob.toSqlflow(userId, token, server, port, time_, dbvendor, sqlfiles) + + if download != '': + while True: + status = GetJobStatus.getStatus(userId, token, server, port, jobId) + if status == 'partial_success' or status == 'success': + GetResultToSqlflow.getResult(download, userId, token, server, port, jobId, time_) + break + + print('========================================grabit-python======================================') + + sys.exit(0) diff --git a/api/python/advanced/SubmitJob.py b/api/python/advanced/SubmitJob.py new file mode 100644 index 0000000..a607996 --- /dev/null +++ b/api/python/advanced/SubmitJob.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import zipfile + +import requests + +import json +import sys +import os + + +def toSqlflow(userId, token, server, port, jobName, dbvendor, sqlfiles): + url = '/gspLive_backend/sqlflow/job/submitUserJob' + + if port != '': + url = server + ':' + port + url + else: + url = server + url + + if os.path.isdir(sqlfiles): + sqlfiles = toZip(sqlfiles) + files = {'sqlfiles': open(sqlfiles, 'rb')} + data = {'dbvendor': dbvendor, 'jobName': jobName, 'token': token, 'userId': userId} + datastr = json.dumps(data) + + print('start submit job to sqlflow.') + + try: + response = requests.post(url, data=eval(datastr), files=files) + except Exception: + print('submit job to sqlflow failed.') + sys.exit(0) + + result = json.loads(response.text) + + if result['code'] == 200: + print('submit job to sqlflow successful.') + return result['data']['jobId'] + else: + print(result['error']) + sys.exit(0) + + +def toZip(start_dir): + if start_dir.endswith(os.sep): + start_dir = start_dir[:-1] + start_dir = start_dir + file_news = start_dir + '.zip' + + z = zipfile.ZipFile(file_news, 'w', zipfile.ZIP_DEFLATED) + for dir_path, dir_names, file_names in os.walk(start_dir): + f_path = dir_path.replace(start_dir, '') + f_path = f_path and f_path + os.sep or '' + for filename in file_names: + z.write(os.path.join(dir_path, filename), f_path + filename) + z.close() + return file_news diff --git a/api/python/advanced/python-data-lineage-overview.png b/api/python/advanced/python-data-lineage-overview.png new file mode 100644 index 0000000..b0c1f5b Binary files /dev/null and b/api/python/advanced/python-data-lineage-overview.png differ diff --git a/api/python/advanced/python-data-lineage-result.json b/api/python/advanced/python-data-lineage-result.json new file mode 100644 index 0000000..edc3291 --- /dev/null +++ b/api/python/advanced/python-data-lineage-result.json @@ -0,0 +1,4973 @@ +{ + "code": 203, + "data": { + "mode": "global", + "summary": { + "schema": 4, + "database": 0, + "view": 2, + "mostRelationTables": [ + { + "schema": "SALES", + "table": "SALESPERSON" + }, + { + "schema": "PERSON", + "table": "PERSON" + }, + { + "table": "VIEW1" + } + ], + "column": 34, + "table": 8, + "relation": 33 + }, + "sqlflow": { + "dbvendor": "dbvmssql", + "dbobjs": [ + { + "id": "54", + "schema": "dbo", + "name": "dbo.uspGetEmployeeSales", + "type": "procedure", + "arguments": [], + "coordinates": [ + { + "x": 9, + "y": 18, + "hashCode": "0" + }, + { + "x": 9, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "8", + "schema": "HumanResources", + "name": "HumanResources.Employee", + "alias": "e", + "type": "table", + "columns": [ + { + "id": "113", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 35, + "y": 28, + "hashCode": "0" + }, + { + "x": 35, + "y": 46, + "hashCode": "0" + }, + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "11", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 34, + "y": 6, + "hashCode": "0" + }, + { + "x": 34, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "13", + "schema": "Person", + "name": "Person.Person", + "alias": "p", + "type": "table", + "columns": [ + { + "id": "118", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 35, + "y": 49, + "hashCode": "0" + }, + { + "x": 35, + "y": 67, + "hashCode": "0" + }, + { + "x": 16, + "y": 34, + "hashCode": "0" + }, + { + "x": 16, + "y": 52, + "hashCode": "0" + }, + { + "x": 25, + "y": 34, + "hashCode": "0" + }, + { + "x": 25, + "y": 52, + "hashCode": "0" + }, + { + "x": 51, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 51, + "hashCode": "0" + }, + { + "x": 53, + "y": 26, + "hashCode": "0" + }, + { + "x": 53, + "y": 44, + "hashCode": "0" + } + ] + }, + { + "id": "15", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "119", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 35, + "y": 6, + "hashCode": "0" + }, + { + "x": 35, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "30", + "name": "FactInternetSales", + "alias": "fis", + "type": "table", + "columns": [ + { + "id": "114", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 44, + "y": 5, + "hashCode": "0" + }, + { + "x": 44, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "32", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "33", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "34", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 42, + "y": 6, + "hashCode": "0" + }, + { + "x": 42, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "37", + "name": "DimSalesTerritory", + "alias": "dst", + "type": "table", + "columns": [ + { + "id": "38", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 44, + "y": 27, + "hashCode": "0" + }, + { + "x": 44, + "y": 48, + "hashCode": "0" + } + ] + }, + { + "id": "39", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 43, + "y": 17, + "hashCode": "0" + }, + { + "x": 43, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "56", + "schema": "Sales", + "name": "Sales.SalesPerson", + "alias": "sp", + "type": "table", + "columns": [ + { + "id": "115", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "116", + "name": "SalesYTD", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "111", + "name": "TerritoryID", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 14, + "y": 10, + "hashCode": "0" + }, + { + "x": 14, + "y": 33, + "hashCode": "0" + } + ] + }, + { + "id": "94", + "schema": "Person", + "name": "Person.Address", + "alias": "a", + "type": "table", + "columns": [ + { + "id": "95", + "name": "AddressID", + "coordinates": [ + { + "x": 53, + "y": 12, + "hashCode": "0" + }, + { + "x": 53, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "117", + "name": "PostalCode", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 52, + "y": 16, + "hashCode": "0" + }, + { + "x": 52, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "2", + "schema": "dbo", + "name": "dbo.EmployeeSales", + "type": "table", + "columns": [ + { + "id": "3", + "name": "DataSource", + "coordinates": [ + { + "x": 3, + "y": 3, + "hashCode": "0" + }, + { + "x": 3, + "y": 13, + "hashCode": "0" + } + ] + }, + { + "id": "4", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 4, + "y": 3, + "hashCode": "0" + }, + { + "x": 4, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "5", + "name": "LastName", + "coordinates": [ + { + "x": 5, + "y": 3, + "hashCode": "0" + }, + { + "x": 5, + "y": 11, + "hashCode": "0" + } + ] + }, + { + "id": "6", + "name": "SalesDollars", + "coordinates": [ + { + "x": 6, + "y": 3, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "87", + "name": "DUMMY0", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "1", + "name": "PseudoRows", + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 31, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "109", + "name": "pseudo_table_include_orphan_column", + "type": "pseudoTable", + "columns": [ + { + "id": "110", + "name": "TerritoryID", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 1, + "y": 1, + "hashCode": "0" + }, + { + "x": 1, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "24", + "name": "hiredate_view", + "type": "view", + "columns": [ + { + "id": "25", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "26", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "27", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "28", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "23", + "name": "PseudoRows", + "coordinates": [ + { + "x": 31, + "y": 13, + "hashCode": "0" + }, + { + "x": 31, + "y": 26, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 31, + "y": 13, + "hashCode": "0" + }, + { + "x": 31, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "48", + "name": "view1", + "type": "view", + "columns": [ + { + "id": "49", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "50", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "51", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "52", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "53", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "47", + "name": "PseudoRows", + "coordinates": [ + { + "x": 38, + "y": 13, + "hashCode": "0" + }, + { + "x": 38, + "y": 18, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 38, + "y": 13, + "hashCode": "0" + }, + { + "x": 38, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "id": "18", + "name": "RS-1", + "type": "select_list", + "columns": [ + { + "id": "19", + "name": "FirstName", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "20", + "name": "LastName", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "21", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + { + "id": "22", + "name": "HireDate", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "17", + "name": "PseudoRows", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + { + "id": "41", + "name": "RS-2", + "type": "select_list", + "columns": [ + { + "id": "42", + "name": "CustomerKey", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "43", + "name": "ProductKey", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + { + "id": "44", + "name": "OrderDateKey", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "45", + "name": "SalesTerritoryKey", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + { + "id": "46", + "name": "SalesTerritoryRegion", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "40", + "name": "PseudoRows", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + { + "id": "66", + "name": "RS-3", + "type": "select_list", + "columns": [ + { + "id": "67", + "name": "PROCEDURE", + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 12, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "69", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + } + ] + }, + { + "id": "70", + "name": "LastName", + "coordinates": [ + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + } + ] + }, + { + "id": "71", + "name": "SalesYTD", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "65", + "name": "PseudoRows", + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 12, + "y": 12, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "81", + "name": "INSERT-SELECT-1", + "type": "insert-select", + "columns": [ + { + "id": "82", + "name": "SELECT", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "84", + "name": "BusinessEntityID", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "85", + "name": "LastName", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + }, + { + "id": "86", + "name": "SalesYTD", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + { + "id": "80", + "name": "PseudoRows", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + { + "id": "98", + "name": "RS-4", + "type": "select_list", + "columns": [ + { + "id": "99", + "name": "\"Row Number\"", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 47, + "y": 89, + "hashCode": "0" + } + ] + }, + { + "id": "105", + "name": "LastName", + "coordinates": [ + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "106", + "name": "SalesYTD", + "coordinates": [ + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + } + ] + }, + { + "id": "107", + "name": "PostalCode", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + }, + { + "id": "97", + "name": "PseudoRows", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + } + ], + "relations": [ + { + "id": "4101", + "type": "fdd", + "effectType": "select", + "target": { + "id": "107", + "column": "PostalCode", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "117", + "column": "PostalCode", + "parentId": "94", + "parentName": "Person.Address", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4102", + "type": "fdd", + "effectType": "select", + "target": { + "id": "69", + "column": "BusinessEntityID", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "115", + "column": "BusinessEntityID", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4103", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "52", + "column": "SalesTerritoryKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "45", + "column": "SalesTerritoryKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4104", + "type": "fdd", + "effectType": "select", + "target": { + "id": "21", + "column": "BusinessEntityID", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "113", + "column": "BusinessEntityID", + "parentId": "8", + "parentName": "HumanResources.Employee", + "coordinates": [ + { + "x": 35, + "y": 28, + "hashCode": "0" + }, + { + "x": 35, + "y": 46, + "hashCode": "0" + }, + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4105", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "26", + "column": "LastName", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "20", + "column": "LastName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4106", + "type": "fdd", + "effectType": "select", + "target": { + "id": "20", + "column": "LastName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4107", + "type": "fdd", + "effectType": "select", + "target": { + "id": "99", + "column": "\"Row Number\"", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 47, + "y": 8, + "hashCode": "0" + }, + { + "x": 47, + "y": 89, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "117", + "column": "PostalCode", + "parentId": "94", + "parentName": "Person.Address", + "coordinates": [ + { + "x": 48, + "y": 29, + "hashCode": "0" + }, + { + "x": 48, + "y": 41, + "hashCode": "0" + }, + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4108", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "28", + "column": "HireDate", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "22", + "column": "HireDate", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4109", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "27", + "column": "BusinessEntityID", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "21", + "column": "BusinessEntityID", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 33, + "hashCode": "0" + }, + { + "x": 33, + "y": 51, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4110", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22", + "column": "HireDate", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "11", + "column": "HireDate", + "parentId": "8", + "parentName": "HumanResources.Employee", + "coordinates": [ + { + "x": 33, + "y": 53, + "hashCode": "0" + }, + { + "x": 33, + "y": 63, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4111", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "4", + "column": "BusinessEntityID", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 4, + "y": 3, + "hashCode": "0" + }, + { + "x": 4, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "84", + "column": "BusinessEntityID", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4112", + "type": "fdd", + "effectType": "select", + "target": { + "id": "106", + "column": "SalesYTD", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4113", + "type": "fdd", + "effectType": "select", + "target": { + "id": "86", + "column": "SalesYTD", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4114", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "51", + "column": "OrderDateKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "44", + "column": "OrderDateKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4115", + "type": "fdd", + "effectType": "select", + "target": { + "id": "70", + "column": "LastName", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4116", + "type": "fdd", + "effectType": "select", + "target": { + "id": "71", + "column": "SalesYTD", + "parentId": "66", + "parentName": "RS-3", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "116", + "column": "SalesYTD", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 13, + "y": 9, + "hashCode": "0" + }, + { + "x": 13, + "y": 20, + "hashCode": "0" + }, + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + }, + { + "x": 48, + "y": 17, + "hashCode": "0" + }, + { + "x": 48, + "y": 27, + "hashCode": "0" + }, + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + }, + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4117", + "type": "fdd", + "effectType": "select", + "target": { + "id": "43", + "column": "ProductKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "33", + "column": "ProductKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4118", + "type": "fdd", + "effectType": "select", + "target": { + "id": "85", + "column": "LastName", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4119", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "50", + "column": "ProductKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "43", + "column": "ProductKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 25, + "hashCode": "0" + }, + { + "x": 40, + "y": 39, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4120", + "type": "fdd", + "effectType": "select", + "target": { + "id": "44", + "column": "OrderDateKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "34", + "column": "OrderDateKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 41, + "hashCode": "0" + }, + { + "x": 40, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4121", + "type": "fdd", + "effectType": "select", + "target": { + "id": "42", + "column": "CustomerKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "32", + "column": "CustomerKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4122", + "type": "fdd", + "effectType": "select", + "target": { + "id": "84", + "column": "BusinessEntityID", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "115", + "column": "BusinessEntityID", + "parentId": "56", + "parentName": "Sales.SalesPerson", + "coordinates": [ + { + "x": 16, + "y": 12, + "hashCode": "0" + }, + { + "x": 16, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + }, + { + "x": 12, + "y": 44, + "hashCode": "0" + }, + { + "x": 17, + "y": 11, + "hashCode": "0" + }, + { + "x": 17, + "y": 30, + "hashCode": "0" + }, + { + "x": 18, + "y": 14, + "hashCode": "0" + }, + { + "x": 18, + "y": 33, + "hashCode": "0" + }, + { + "x": 25, + "y": 12, + "hashCode": "0" + }, + { + "x": 25, + "y": 31, + "hashCode": "0" + }, + { + "x": 22, + "y": 22, + "hashCode": "0" + }, + { + "x": 22, + "y": 41, + "hashCode": "0" + }, + { + "x": 26, + "y": 11, + "hashCode": "0" + }, + { + "x": 26, + "y": 30, + "hashCode": "0" + }, + { + "x": 27, + "y": 14, + "hashCode": "0" + }, + { + "x": 27, + "y": 33, + "hashCode": "0" + }, + { + "x": 51, + "y": 12, + "hashCode": "0" + }, + { + "x": 51, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4123", + "type": "fdd", + "effectType": "select", + "target": { + "id": "19", + "column": "FirstName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "15", + "column": "FirstName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4124", + "type": "fdd", + "effectType": "select", + "target": { + "id": "105", + "column": "LastName", + "parentId": "98", + "parentName": "RS-4", + "coordinates": [ + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "119", + "column": "LastName", + "parentId": "13", + "parentName": "Person.Person", + "coordinates": [ + { + "x": 33, + "y": 21, + "hashCode": "0" + }, + { + "x": 33, + "y": 31, + "hashCode": "0" + }, + { + "x": 12, + "y": 46, + "hashCode": "0" + }, + { + "x": 12, + "y": 56, + "hashCode": "0" + }, + { + "x": 18, + "y": 35, + "hashCode": "0" + }, + { + "x": 18, + "y": 45, + "hashCode": "0" + }, + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + }, + { + "x": 27, + "y": 35, + "hashCode": "0" + }, + { + "x": 27, + "y": 45, + "hashCode": "0" + }, + { + "x": 48, + "y": 5, + "hashCode": "0" + }, + { + "x": 48, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4125", + "type": "fdd", + "effectType": "select", + "target": { + "id": "45", + "column": "SalesTerritoryKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "114", + "column": "SalesTerritoryKey", + "parentId": "30", + "parentName": "FactInternetSales", + "coordinates": [ + { + "x": 44, + "y": 5, + "hashCode": "0" + }, + { + "x": 44, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 3, + "hashCode": "0" + }, + { + "x": 41, + "y": 24, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4126", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "49", + "column": "CustomerKey", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "42", + "column": "CustomerKey", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 40, + "y": 8, + "hashCode": "0" + }, + { + "x": 40, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4127", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "5", + "column": "LastName", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 5, + "y": 3, + "hashCode": "0" + }, + { + "x": 5, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "85", + "column": "LastName", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 43, + "hashCode": "0" + }, + { + "x": 22, + "y": 53, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4128", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "6", + "column": "SalesDollars", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 6, + "y": 3, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "86", + "column": "SalesYTD", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 55, + "hashCode": "0" + }, + { + "x": 22, + "y": 66, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4129", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "25", + "column": "FirstName", + "parentId": "24", + "parentName": "hiredate_view", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "19", + "column": "FirstName", + "parentId": "18", + "parentName": "RS-1", + "coordinates": [ + { + "x": 33, + "y": 8, + "hashCode": "0" + }, + { + "x": 33, + "y": 19, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4130", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "87", + "column": "DUMMY0", + "parentId": "2", + "parentName": "dbo.EmployeeSales", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "82", + "column": "SELECT", + "parentId": "81", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 22, + "y": 12, + "hashCode": "0" + }, + { + "x": 22, + "y": 20, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4131", + "type": "fdd", + "effectType": "create_view", + "target": { + "id": "53", + "column": "SalesTerritoryRegion", + "parentId": "48", + "parentName": "view1", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "46", + "column": "SalesTerritoryRegion", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4132", + "type": "fdd", + "effectType": "select", + "target": { + "id": "46", + "column": "SalesTerritoryRegion", + "parentId": "41", + "parentName": "RS-2", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "39", + "column": "SalesTerritoryRegion", + "parentId": "37", + "parentName": "DimSalesTerritory", + "coordinates": [ + { + "x": 41, + "y": 26, + "hashCode": "0" + }, + { + "x": 41, + "y": 50, + "hashCode": "0" + } + ] + } + ] + } + ], + "errors": [ + { + "errorMessage": "find orphan column(10500) near: PostalCode(47,39)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 47, + "y": 39, + "hashCode": "0" + }, + { + "x": 47, + "y": 49, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: SalesYTD(47,59)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 47, + "y": 59, + "hashCode": "0" + }, + { + "x": 47, + "y": 67, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: TerritoryID(54,7)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: SalesYTD(55,9)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 55, + "y": 9, + "hashCode": "0" + }, + { + "x": 55, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "find orphan column(10500) near: PostalCode(56,10)", + "errorType": "SyntaxHint", + "coordinates": [ + { + "x": 56, + "y": 10, + "hashCode": "0" + }, + { + "x": 56, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "errorMessage": "Link orphan column [TerritoryID] to the first table [Sales.SalesPerson s]", + "errorType": "LinkOrphanColumn", + "coordinates": [ + { + "x": 54, + "y": 7, + "hashCode": "0" + }, + { + "x": 54, + "y": 18, + "hashCode": "0" + } + ] + } + ] + }, + "graph": { + "elements": { + "tables": [ + { + "columns": [ + { + "height": 16, + "id": "n0::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -625.07874 + }, + { + "height": 16, + "id": "n0::n1", + "label": { + "content": "HIREDATE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 76, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -609.07874 + } + ], + "height": 57.96875, + "id": "n0", + "label": { + "content": "HumanResources.Employee", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 178, + "x": -8, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -647.0475 + }, + { + "columns": [ + { + "height": 16, + "id": "n1::n0", + "label": { + "content": "FIRSTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 84, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -537.1097 + }, + { + "height": 16, + "id": "n1::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -521.1097 + } + ], + "height": 57.96875, + "id": "n1", + "label": { + "content": "Person.Person", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -559.0784 + }, + { + "columns": [ + { + "height": 16, + "id": "n2::n0", + "label": { + "content": "SALESTERRITORYKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 141, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -78.00015 + }, + { + "height": 16, + "id": "n2::n1", + "label": { + "content": "CUSTOMERKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 105, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -62.00015 + }, + { + "height": 16, + "id": "n2::n2", + "label": { + "content": "PRODUCTKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -46.00015 + }, + { + "height": 16, + "id": "n2::n3", + "label": { + "content": "ORDERDATEKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 112, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -30.000149 + } + ], + "height": 89.96875, + "id": "n2", + "label": { + "content": "FactInternetSales", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -99.9689 + }, + { + "columns": [ + { + "height": 16, + "id": "n3::n0", + "label": { + "content": "SALESTERRITORYREGION", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 165, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -149.9692 + } + ], + "height": 41.96875, + "id": "n3", + "label": { + "content": "DimSalesTerritory", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -171.93794 + }, + { + "columns": [ + { + "height": 16, + "id": "n4::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -362.57196 + }, + { + "height": 16, + "id": "n4::n1", + "label": { + "content": "SALESYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 77, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -346.57196 + } + ], + "height": 57.96875, + "id": "n4", + "label": { + "content": "Sales.SalesPerson", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -384.5407 + }, + { + "columns": [ + { + "height": 16, + "id": "n5::n0", + "label": { + "content": "POSTALCODE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -221.93825 + } + ], + "height": 41.96875, + "id": "n5", + "label": { + "content": "Person.Address", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -243.907 + }, + { + "columns": [ + { + "height": 16, + "id": "n6::n0", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -495.37158 + }, + { + "height": 16, + "id": "n6::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -479.37158 + }, + { + "height": 16, + "id": "n6::n2", + "label": { + "content": "SALESDOLLARS", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 108, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -463.37158 + }, + { + "height": 16, + "id": "n6::n3", + "label": { + "content": "DUMMY0", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 69, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -447.37158 + } + ], + "height": 89.96875, + "id": "n6", + "label": { + "content": "dbo.EmployeeSales", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -517.34033 + }, + { + "columns": [ + { + "height": 16, + "id": "n7::n0", + "label": { + "content": "FIRSTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 84, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -615.34064 + }, + { + "height": 16, + "id": "n7::n1", + "label": { + "content": "LASTNAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -599.34064 + }, + { + "height": 16, + "id": "n7::n2", + "label": { + "content": "BUSINESSENTITYID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 130, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -583.34064 + }, + { + "height": 16, + "id": "n7::n3", + "label": { + "content": "HIREDATE", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 76, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -567.34064 + } + ], + "height": 89.96875, + "id": "n7", + "label": { + "content": "hiredate_view", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -637.3094 + }, + { + "columns": [ + { + "height": 16, + "id": "n8::n0", + "label": { + "content": "CUSTOMERKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 105, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -97.24618 + }, + { + "height": 16, + "id": "n8::n1", + "label": { + "content": "PRODUCTKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 96, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -81.24618 + }, + { + "height": 16, + "id": "n8::n2", + "label": { + "content": "ORDERDATEKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 112, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -65.24618 + }, + { + "height": 16, + "id": "n8::n3", + "label": { + "content": "SALESTERRITORYKEY", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 141, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -49.24618 + }, + { + "height": 16, + "id": "n8::n4", + "label": { + "content": "SALESTERRITORYREGION", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 165, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 485, + "y": -33.24618 + } + ], + "height": 105.96875, + "id": "n8", + "label": { + "content": "view1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 484, + "y": -119.21493 + }, + { + "columns": [ + { + "height": 16, + "id": "n9::n0", + "label": { + "content": "FirstName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 79, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -615.34064 + }, + { + "height": 16, + "id": "n9::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -599.34064 + }, + { + "height": 16, + "id": "n9::n2", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -583.34064 + }, + { + "height": 16, + "id": "n9::n3", + "label": { + "content": "HireDate", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 71, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -567.34064 + } + ], + "height": 89.96875, + "id": "n9", + "label": { + "content": "RS-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -637.3094 + }, + { + "columns": [ + { + "height": 16, + "id": "n10::n0", + "label": { + "content": "CustomerKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 97, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -97.24618 + }, + { + "height": 16, + "id": "n10::n1", + "label": { + "content": "ProductKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 86, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -81.24618 + }, + { + "height": 16, + "id": "n10::n2", + "label": { + "content": "OrderDateKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 102, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -65.24618 + }, + { + "height": 16, + "id": "n10::n3", + "label": { + "content": "SalesTerritoryKey", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 123, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -49.24618 + }, + { + "height": 16, + "id": "n10::n4", + "label": { + "content": "SalesTerritoryRegion", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 145, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -33.24618 + } + ], + "height": 105.96875, + "id": "n10", + "label": { + "content": "RS-2", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -119.21493 + }, + { + "columns": [ + { + "height": 16, + "id": "n11::n0", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -375.40253 + }, + { + "height": 16, + "id": "n11::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -359.40253 + }, + { + "height": 16, + "id": "n11::n2", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -343.40253 + } + ], + "height": 73.96875, + "id": "n11", + "label": { + "content": "RS-3", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -397.37128 + }, + { + "columns": [ + { + "height": 16, + "id": "n12::n0", + "label": { + "content": "SELECT", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -495.37158 + }, + { + "height": 16, + "id": "n12::n1", + "label": { + "content": "BusinessEntityID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 119, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -479.37158 + }, + { + "height": 16, + "id": "n12::n2", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -463.37158 + }, + { + "height": 16, + "id": "n12::n3", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -447.37158 + } + ], + "height": 89.96875, + "id": "n12", + "label": { + "content": "INSERT-SELECT-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -517.34033 + }, + { + "columns": [ + { + "height": 16, + "id": "n13::n0", + "label": { + "content": "\"Row Number\"", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 104, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -271.43347 + }, + { + "height": 16, + "id": "n13::n1", + "label": { + "content": "LastName", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 78, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -255.43349 + }, + { + "height": 16, + "id": "n13::n2", + "label": { + "content": "SalesYTD", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 73, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -239.43349 + }, + { + "height": 16, + "id": "n13::n3", + "label": { + "content": "PostalCode", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 87, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 273, + "y": -223.43349 + } + ], + "height": 89.96875, + "id": "n13", + "label": { + "content": "RS-4", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 272, + "y": -293.40222 + } + ], + "edges": [ + { + "id": "e0", + "sourceId": "n4::n0", + "targetId": "n12::n1" + }, + { + "id": "e1", + "sourceId": "n9::n1", + "targetId": "n7::n1" + }, + { + "id": "e2", + "sourceId": "n2::n3", + "targetId": "n10::n2" + }, + { + "id": "e3", + "sourceId": "n9::n3", + "targetId": "n7::n3" + }, + { + "id": "e4", + "sourceId": "n1::n1", + "targetId": "n13::n1" + }, + { + "id": "e5", + "sourceId": "n1::n1", + "targetId": "n12::n2" + }, + { + "id": "e6", + "sourceId": "n10::n4", + "targetId": "n8::n4" + }, + { + "id": "e7", + "sourceId": "n5::n0", + "targetId": "n13::n3" + }, + { + "id": "e8", + "sourceId": "n2::n2", + "targetId": "n10::n1" + }, + { + "id": "e9", + "sourceId": "n1::n1", + "targetId": "n11::n1" + }, + { + "id": "e10", + "sourceId": "n12::n3", + "targetId": "n6::n2" + }, + { + "id": "e11", + "sourceId": "n2::n1", + "targetId": "n10::n0" + }, + { + "id": "e12", + "sourceId": "n4::n1", + "targetId": "n11::n2" + }, + { + "id": "e13", + "sourceId": "n12::n2", + "targetId": "n6::n1" + }, + { + "id": "e14", + "sourceId": "n1::n1", + "targetId": "n9::n1" + }, + { + "id": "e15", + "sourceId": "n12::n1", + "targetId": "n6::n0" + }, + { + "id": "e16", + "sourceId": "n1::n0", + "targetId": "n9::n0" + }, + { + "id": "e17", + "sourceId": "n9::n2", + "targetId": "n7::n2" + }, + { + "id": "e18", + "sourceId": "n4::n1", + "targetId": "n12::n3" + }, + { + "id": "e19", + "sourceId": "n0::n1", + "targetId": "n9::n3" + }, + { + "id": "e20", + "sourceId": "n9::n0", + "targetId": "n7::n0" + }, + { + "id": "e21", + "sourceId": "n3::n0", + "targetId": "n10::n4" + }, + { + "id": "e22", + "sourceId": "n12::n0", + "targetId": "n6::n3" + }, + { + "id": "e23", + "sourceId": "n2::n0", + "targetId": "n10::n3" + }, + { + "id": "e24", + "sourceId": "n0::n0", + "targetId": "n9::n2" + }, + { + "id": "e25", + "sourceId": "n10::n2", + "targetId": "n8::n2" + }, + { + "id": "e26", + "sourceId": "n4::n1", + "targetId": "n13::n2" + }, + { + "id": "e27", + "sourceId": "n4::n1", + "targetId": "n13::n0" + }, + { + "id": "e28", + "sourceId": "n5::n0", + "targetId": "n13::n0" + }, + { + "id": "e29", + "sourceId": "n10::n1", + "targetId": "n8::n1" + }, + { + "id": "e30", + "sourceId": "n10::n0", + "targetId": "n8::n0" + }, + { + "id": "e31", + "sourceId": "n10::n3", + "targetId": "n8::n3" + }, + { + "id": "e32", + "sourceId": "n4::n0", + "targetId": "n11::n0" + } + ] + }, + "tooltip": { + "pseudo_tab...rphan_column": "pseudo_table_include_orphan_column" + }, + "relationIdMap": { + "e0": "fdd", + "e1": "fdd", + "e2": "fdd", + "e3": "fdd", + "e4": "fdd", + "e5": "fdd", + "e6": "fdd", + "e7": "fdd", + "e8": "fdd", + "e9": "fdd", + "e10": "fdd", + "e11": "fdd", + "e12": "fdd", + "e13": "fdd", + "e14": "fdd", + "e15": "fdd", + "e16": "fdd", + "e17": "fdd", + "e18": "fdd", + "e19": "fdd", + "e20": "fdd", + "e21": "fdd", + "e22": "fdd", + "e23": "fdd", + "e24": "fdd", + "e25": "fdd", + "e26": "fdd", + "e27": "fdd", + "e28": "fdd", + "e29": "fdd", + "e30": "fdd", + "e31": "fdd", + "e32": "fdd" + }, + "listIdMap": { + "n0": [ + "8" + ], + "n0::n0": [ + "113" + ], + "n0::n1": [ + "11" + ], + "n1": [ + "13" + ], + "n1::n0": [ + "15" + ], + "n1::n1": [ + "119" + ], + "n2": [ + "30" + ], + "n2::n0": [ + "114" + ], + "n2::n1": [ + "32" + ], + "n2::n2": [ + "33" + ], + "n2::n3": [ + "34" + ], + "n3": [ + "37" + ], + "n3::n0": [ + "39" + ], + "n4": [ + "56" + ], + "n4::n0": [ + "115" + ], + "n4::n1": [ + "116" + ], + "n5": [ + "94" + ], + "n5::n0": [ + "117" + ], + "n6": [ + "2" + ], + "n6::n0": [ + "4" + ], + "n6::n1": [ + "5" + ], + "n6::n2": [ + "6" + ], + "n6::n3": [ + "87" + ], + "n7": [ + "24" + ], + "n7::n0": [ + "25" + ], + "n7::n1": [ + "26" + ], + "n7::n2": [ + "27" + ], + "n7::n3": [ + "28" + ], + "n8": [ + "48" + ], + "n8::n0": [ + "49" + ], + "n8::n1": [ + "50" + ], + "n8::n2": [ + "51" + ], + "n8::n3": [ + "52" + ], + "n8::n4": [ + "53" + ], + "n9": [ + "18" + ], + "n9::n0": [ + "19" + ], + "n9::n1": [ + "20" + ], + "n9::n2": [ + "21" + ], + "n9::n3": [ + "22" + ], + "n10": [ + "41" + ], + "n10::n0": [ + "42" + ], + "n10::n1": [ + "43" + ], + "n10::n2": [ + "44" + ], + "n10::n3": [ + "45" + ], + "n10::n4": [ + "46" + ], + "n11": [ + "66" + ], + "n11::n0": [ + "69" + ], + "n11::n1": [ + "70" + ], + "n11::n2": [ + "71" + ], + "n12": [ + "81" + ], + "n12::n0": [ + "82" + ], + "n12::n1": [ + "84" + ], + "n12::n2": [ + "85" + ], + "n12::n3": [ + "86" + ], + "n13": [ + "98" + ], + "n13::n0": [ + "99" + ], + "n13::n1": [ + "105" + ], + "n13::n2": [ + "106" + ], + "n13::n3": [ + "107" + ] + } + } + }, + "error": "find orphan column(10500) near: PostalCode(47,39)\r\nfind orphan column(10500) near: SalesYTD(47,59)\r\nfind orphan column(10500) near: TerritoryID(54,7)\r\nfind orphan column(10500) near: SalesYTD(55,9)\r\nfind orphan column(10500) near: PostalCode(56,10)\r\nLink orphan column [TerritoryID] to the first table [Sales.SalesPerson s]", + "sessionId": "ec3938ae68a8a468d729e6c426108e701e78079cf070a385acf7cd899d41058c_1615189437331" +} \ No newline at end of file diff --git a/api/python/advanced/python-data-lineage-sqlserver.sql b/api/python/advanced/python-data-lineage-sqlserver.sql new file mode 100644 index 0000000..d8da974 --- /dev/null +++ b/api/python/advanced/python-data-lineage-sqlserver.sql @@ -0,0 +1,56 @@ +-- sql server sample sql +CREATE TABLE dbo.EmployeeSales +( DataSource varchar(20) NOT NULL, + BusinessEntityID varchar(11) NOT NULL, + LastName varchar(40) NOT NULL, + SalesDollars money NOT NULL +); +GO +CREATE PROCEDURE dbo.uspGetEmployeeSales +AS + SET NOCOUNT ON; + SELECT 'PROCEDURE', sp.BusinessEntityID, c.LastName, + sp.SalesYTD + FROM Sales.SalesPerson AS sp + INNER JOIN Person.Person AS c + ON sp.BusinessEntityID = c.BusinessEntityID + WHERE sp.BusinessEntityID LIKE '2%' + ORDER BY sp.BusinessEntityID, c.LastName; +GO +--INSERT...SELECT example +INSERT INTO dbo.EmployeeSales + SELECT 'SELECT', sp.BusinessEntityID, c.LastName, sp.SalesYTD + FROM Sales.SalesPerson AS sp + INNER JOIN Person.Person AS c + ON sp.BusinessEntityID = c.BusinessEntityID + WHERE sp.BusinessEntityID LIKE '2%' + ORDER BY sp.BusinessEntityID, c.LastName; +GO + + +CREATE VIEW hiredate_view +AS +SELECT p.FirstName, p.LastName, e.BusinessEntityID, e.HireDate +FROM HumanResources.Employee e +JOIN Person.Person AS p ON e.BusinessEntityID = p.BusinessEntityID ; +GO + +CREATE VIEW view1 +AS +SELECT fis.CustomerKey, fis.ProductKey, fis.OrderDateKey, + fis.SalesTerritoryKey, dst.SalesTerritoryRegion +FROM FactInternetSales AS fis +LEFT OUTER JOIN DimSalesTerritory AS dst +ON (fis.SalesTerritoryKey=dst.SalesTerritoryKey); + +GO +SELECT ROW_NUMBER() OVER(PARTITION BY PostalCode ORDER BY SalesYTD DESC) AS "Row Number", + p.LastName, s.SalesYTD, a.PostalCode +FROM Sales.SalesPerson AS s + INNER JOIN Person.Person AS p + ON s.BusinessEntityID = p.BusinessEntityID + INNER JOIN Person.Address AS a + ON a.AddressID = p.BusinessEntityID +WHERE TerritoryID IS NOT NULL + AND SalesYTD <> 0 +ORDER BY PostalCode; \ No newline at end of file diff --git a/api/python/advanced/python-data-lineage.png b/api/python/advanced/python-data-lineage.png new file mode 100644 index 0000000..14955be Binary files /dev/null and b/api/python/advanced/python-data-lineage.png differ diff --git a/api/python/advanced/readme.md b/api/python/advanced/readme.md new file mode 100644 index 0000000..3bf8996 --- /dev/null +++ b/api/python/advanced/readme.md @@ -0,0 +1,129 @@ +## Python Data lineage: using the SQLFlow REST API (Advanced) + +This article illustrates how to discover the data lineage using Python and the SQLFlow REST API. + +By using the SQLFlow REST API, you can code in python to discover the data lineage in SQL scripts +and get the result in an actionable diagram, json, csv or graphml format. + +You can integerate the python code provided here into your own project and add the powerful +data lineage analsysis capability instantly. + +### 1. interactive data lineage visualizations +![Python Data lineage](python-data-lineage.png) + +### 2. [Data lineage in JSON format](python-data-lineage-result.json) + +### 3. Data lineage in CSV, graphml format + + +## Prerequisites +- [SQLFlow Cloud Server or on-premise version](https://github.com/sqlparser/sqlflow_public/tree/master/api#prerequisites) +- Python 2.7 or higher version must be installed and configured correctly. +- Installing Dependency Libraries: +``` +pip install requests +``` + +### Usage +```` +python Grabit.py /s server /p port /u userId /k userSecret /t databaseType /f path_to_config_file /r resultType + +eg: + python Grabit.py /u 'auth0|xxx' /k cab9712c45189014a94a8b7aceeef7a3db504be58e18cd3686f3bbefd078ef4d /s https://api.gudusoft.com /t oracle /f demo.sql /r 1 + +note: + If the parameter string contains symbols like "|" , it must be included in a single quotes (' ') +```` + +Example: + +1. Connect to the SQLFlow Cloud Server +``` +python Grabit.py /s https://api.gudusoft.com /u 'YOUR_USER_ID' /k YOUR_SECRET_KEY /t sqlserver /f python-data-lineage-sqlserver.sql /r 1 +``` + +2. Connect to the SQLFlow on-premise +This will discover data lineage by analyzing the `python-data-lineage-sqlserver.sql` file. You may also specify a zip file which includes lots of SQL files. +``` +python Grabit.py /s http://127.0.0.1 /p 8081 /u 'gudu|0123456789' /t sqlserver /f python-data-lineage-sqlserver.sql /r 1 +``` + +This will discover data lineage by analyzing all SQL files under `sqlfiles` directory. +``` +python Grabit.py /s http://127.0.0.1 /p 8081 /u 'gudu|0123456789' /t mysql /f sqlfiles /r 1 +``` + + +### Parameters + +- **path_to_config_file** + +This can be a single SQL file, a zip file including multiple SQL files, or a directory including lots of SQL files. + +- **server** + +Usually, it is the IP address of [the SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +installed on your owner servers such as `127.0.0.1` or `http://127.0.0.1` + +You may set the value to `https://api.gudusoft.com` if you like to send your SQL script to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com) to get the data lineage result. + +- **port** + +The default value is `8081` if you connect to your SQLFlow on-premise server. + +However, if you setup the nginx reverse proxy in the nginx configuration file like this: +``` + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } +``` +Then, keep the value of `serverPort` empty and set `server` to the value like this: `http://127.0.0.1/api`. + +>Please keep this value empty if you connect to the SQLFlow Cloud Server by specifying the `https://api.gudusoft.com` +in the `server` + > +- **userId, userSecret** + +This is the user id that is used to connect to the SQLFlow server. +Always set this value to `gudu|0123456789` and keep `userSecret` empty if you use the SQLFlow on-premise version. + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + + +- **databaseType** + +This parameter specifies the database dialect of the SQL scripts that the SQLFlow has analyzed. + +```txt + access,bigquery,couchbase,dax,db2,greenplum,hana,hive,impala,informix,mdx,mssql, + sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake, + sybase,teradata,soql,vertica +``` + +- **resultType** + +When you submit SQL script to the SQLFlow server, A job is created on the SQLFlow server +and you can always see the graphic data lineage result via the browser, + + +Even better, This demo will fetch the data lineage back to the directory where the demo is running. +Those data lineage results are stored in the `data/result/` directory. + +This parameter specifies which kind of format is used to save the data lineage result. + +Available values for this parameter: +- 1: JSON, data lineage result in JSON. +- 2: CSV, data lineage result in CSV format. +- 3: diagram, in graphml format that can be viewed by yEd. + +### SQLFlow REST API +Please check here for the detailed information about the [SQLFlow REST API](https://github.com/sqlparser/sqlflow_public/tree/master/api/sqlflow_api.md) diff --git a/api/python/basic/GenerateDataLineageDemo.py b/api/python/basic/GenerateDataLineageDemo.py new file mode 100644 index 0000000..f2c0fe6 --- /dev/null +++ b/api/python/basic/GenerateDataLineageDemo.py @@ -0,0 +1,232 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import zipfile + +import requests +import time +import json +import sys +import os + + +def toSqlflow(userId, token, server, port, jobName, dbvendor, sqlfiles): + url = '/api/gspLive_backend/sqlflow/job/submitUserJob' + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/job/submitUserJob' + + if port != '': + url = server + ':' + port + url + else: + url = server + url + + if os.path.isdir(sqlfiles): + sqlfiles = toZip(sqlfiles) + files = {'sqlfiles': open(sqlfiles, 'rb')} + data = {'dbvendor': dbvendor, 'jobName': jobName, 'token': token, 'userId': userId} + datastr = json.dumps(data) + + print('start submit job to sqlflow.') + + try: + response = requests.post(url, data=eval(datastr), files=files, verify=False) + except Exception: + print('submit job to sqlflow failed.') + sys.exit(0) + + result = json.loads(response.text) + + if result['code'] == 200: + print('submit job to sqlflow successful.') + return result['data']['jobId'] + else: + print(result['error']) + sys.exit(0) + + +def toZip(start_dir): + if start_dir.endswith(os.sep): + start_dir = start_dir[:-1] + start_dir = start_dir + file_news = start_dir + '.zip' + + z = zipfile.ZipFile(file_news, 'w', zipfile.ZIP_DEFLATED) + for dir_path, dir_names, file_names in os.walk(start_dir): + f_path = dir_path.replace(start_dir, '') + f_path = f_path and f_path + os.sep or '' + for filename in file_names: + z.write(os.path.join(dir_path, filename), f_path + filename) + z.close() + return file_news + + +def getToken(userId, server, port,screctKey): + + if userId == 'gudu|0123456789': + return 'token' + + url = '/api/gspLive_backend/user/generateToken' + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/user/generateToken' + if port != '': + url = server + ':' + port + url + else: + url = server + url + mapA = {'secretKey': screctKey, 'userId': userId} + header_dict = {"Content-Type": "application/x-www-form-urlencoded"} + + print('start get token.') + try: + r = requests.post(url, data=mapA, headers=header_dict, verify=False) + print(r) + except Exception: + print('get token failed.') + result = json.loads(r.text) + + if result['code'] == '200': + print('get token successful.') + return result['token'] + else: + print(result['error']) + + +def getResult(dataLineageFileType, userId, token, server, port, jobId, filePath): + sep = 'data' + os.sep + 'result' + os.sep + filePath = filePath + '_' + jobId + if dataLineageFileType == 'json': + url = "/api/gspLive_backend/sqlflow/job/exportLineageAsJson" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/job/exportLineageAsJson' + filePath = sep + filePath + '_json.json' + elif dataLineageFileType == 'graphml': + url = "/api/gspLive_backend/sqlflow/job/exportLineageAsGraphml" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/job/exportLineageAsGraphml' + filePath = sep + filePath + '_graphml.graphml' + elif dataLineageFileType == 'csv': + url = "/api/gspLive_backend/sqlflow/job/exportLineageAsCsv" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/job/exportLineageAsCsv' + filePath = sep + filePath + '_csv.csv' + else: + url = "/api/gspLive_backend/sqlflow/job/exportLineageAsJson" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/job/exportLineageAsJson' + filePath = sep + filePath + '_json.json' + + if port != '': + url = server + ':' + port + url + else: + url = server + url + + data = {'jobId': jobId, 'token': token, 'userId': userId, 'tableToTable': 'false'} + datastr = json.dumps(data) + + print('start download result to sqlflow.') + try: + response = requests.post(url, data=eval(datastr), verify=False) + except Exception: + print('download result to sqlflow failed.') + sys.exit(0) + + if not os.path.exists(sep): + os.makedirs(sep) + + try: + with open(filePath, 'wb') as f: + f.write(response.content) + except Exception: + print(filePath, 'is not exist.') + sys.exit(0) + + print('download result to sqlflow successful.file path is ', filePath) + + + +def getStatus(userId, token, server, port, jobId): + url = "/api/gspLive_backend/sqlflow/job/displayUserJobSummary" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/job/displayUserJobSummary' + + if port != '': + url = server + ':' + port + url + else: + url = server + url + + data = {'jobId': jobId, 'token': token, 'userId': userId} + datastr = json.dumps(data) + + try: + response = requests.post(url, data=eval(datastr), verify=False) + except Exception: + print('get job status to sqlflow failed.') + sys.exit(0) + + result = json.loads(response.text) + if result['code'] == 200: + status = result['data']['status'] + if status == 'fail': + print(result['data']['errorMessage']) + sys.exit(0) + return status + + +if __name__ == '__main__': + if len(sys.argv) < 1: + print('Please enter the args.') + sys.exit(0) + + # the user id of sqlflow web or client, required true + userId = '' + + # the secret key of sqlflow user for webapi request, required true + screctKey = '' + + + # sqlflow server + server = '' + + # sqlflow api port + port = '' + + # database type + dbvendor = 'dbvmysql' + + sqlfile = '' + dataLineageFileType = '' + for i in range(1, len(sys.argv)): + if sys.argv[i] == '/f': + try: + if sys.argv[i + 1] is not None: + sqlfile = sys.argv[i + 1] + except Exception: + print('Please enter the sqlfile path，required true. eg: /f sql.txt') + sys.exit(0) + elif sys.argv[i] == '/o': + try: + if sys.argv[i + 1] is not None: + dataLineageFileType = sys.argv[i + 1] + except Exception: + dataLineageFileType = 'json' + + token = getToken(userId, server, port, screctKey); + + # sqlflow job name + jobName = 'test' + jobId = toSqlflow(userId, token, server, port, jobName, dbvendor, sqlfile) + + while 1==1: + status = getStatus(userId, token, server, port, jobId) + if status == 'fail': + print('job execute failed.') + break; + elif status == 'success': + print('job execute successful.') + break; + elif status == 'partial_success': + print('job execute partial successful.') + break; + time.sleep(2) + + # data lineage file path + filePath = 'datalineage' + getResult(dataLineageFileType, userId, token, server, port, jobId, filePath) diff --git a/api/python/basic/GenerateLineageParam.py b/api/python/basic/GenerateLineageParam.py new file mode 100644 index 0000000..2772b50 --- /dev/null +++ b/api/python/basic/GenerateLineageParam.py @@ -0,0 +1,57 @@ +import zipfile +import sys +import os + + +def toZip(start_dir): + if start_dir.endswith(os.sep): + start_dir = start_dir[:-1] + start_dir = start_dir + file_news = start_dir + '.zip' + + z = zipfile.ZipFile(file_news, 'w', zipfile.ZIP_DEFLATED) + for dir_path, dir_names, file_names in os.walk(start_dir): + f_path = dir_path.replace(start_dir, '') + f_path = f_path and f_path + os.sep or '' + for filename in file_names: + z.write(os.path.join(dir_path, filename), f_path + filename) + z.close() + return file_news + + +def buildSqltextParam(userId, token, delimiter, export_include_table, showConstantTable, + treatArgumentsInCountFunctionAsDirectDataflow, dbvendor, sqltext): + data = {'dbvendor': dbvendor, 'token': token, 'userId': userId} + if delimiter != '': + data['delimiter'] = delimiter + if export_include_table != '': + data['export_include_table'] = export_include_table + if showConstantTable != '': + data['showConstantTable'] = showConstantTable + if treatArgumentsInCountFunctionAsDirectDataflow != '': + data['treatArgumentsInCountFunctionAsDirectDataflow'] = treatArgumentsInCountFunctionAsDirectDataflow + if sqltext != '': + data['sqltext'] = sqltext + return data + + +def buildSqlfileParam(userId, token, delimiter, export_include_table, showConstantTable, + treatArgumentsInCountFunctionAsDirectDataflow, dbvendor, sqlfile): + files = '' + if sqlfile != '': + if os.path.isdir(sqlfile): + print('The SQL file cannot be a directory.') + sys.exit(0) + files = {'sqlfile': open(sqlfile, 'rb')} + + data = {'dbvendor': dbvendor, 'token': token, 'userId': userId} + if delimiter != '': + data['delimiter'] = delimiter + if export_include_table != '': + data['export_include_table'] = export_include_table + if showConstantTable != '': + data['showConstantTable'] = showConstantTable + if treatArgumentsInCountFunctionAsDirectDataflow != '': + data['treatArgumentsInCountFunctionAsDirectDataflow'] = treatArgumentsInCountFunctionAsDirectDataflow + return data, files + diff --git a/api/python/basic/GenerateToken.py b/api/python/basic/GenerateToken.py new file mode 100644 index 0000000..37acd45 --- /dev/null +++ b/api/python/basic/GenerateToken.py @@ -0,0 +1,44 @@ +import requests + +import json + +def getToken(userId, server, port, screctKey): + if userId == 'gudu|0123456789': + return 'token' + url = '/api/gspLive_backend/user/generateToken' + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/user/generateToken' + if port != '': + url = server + ':' + port + url + else: + url = server + url + mapA = {'secretKey': screctKey, 'userId': userId} + header_dict = {"Content-Type": "application/x-www-form-urlencoded"} + + try: + r = requests.post(url, data=mapA, headers=header_dict, verify=False) + except Exception as e: + print('get token failed.', e) + result = json.loads(r.text) + + if result['code'] == '200': + return result['token'] + else: + print(result['error']) + + +if __name__ == '__main__': + + server = '' + + port = '' + + # the user id of sqlflow web or client, required true + userId = '' + + # the secret key of sqlflow user for webapi request, required true + screctKey = '' + + token = getToken(userId, server, port, screctKey) + + print(token) diff --git a/api/python/basic/checksyntax.py b/api/python/basic/checksyntax.py new file mode 100644 index 0000000..231188b --- /dev/null +++ b/api/python/basic/checksyntax.py @@ -0,0 +1,61 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import requests +import json +import GenerateToken + + +def check(server, port, sql, dbvendor, userId, token): + url = "/api/gspLive_backend/demo/syntax/check" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/demo/syntax/check' + if port != '': + url = server + ':' + port + url + else: + url = server + url + + data = {'sql': sql, 'dbvendor': dbvendor, 'userId': userId, 'token': token} + header_dict = {"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"} + try: + r = requests.post(url, data=data, headers=header_dict, verify=False) + except Exception as e: + print('syntax error.', e) + result = json.loads(r.text) + + if result['code'] == 200: + usedTime = result['data']['usedTime'] + version = result['data']['gsp.version'] + print('syntax correct. elapsed time: ' + usedTime+' ,gsp version: ' + version) + else: + usedTime = result['data']['usedTime'] + version = result['data']['gsp.version'] + print('syntax error. elapsed time: ' + usedTime + ' ,gsp version: ' + version + ' ,error info:') + errorInfos = result['data']['errorInfos'] + for error in errorInfos: + print(error['errorMessage']) + +if __name__ == '__main__': + # the user id of sqlflow web or client, required true + userId = '' + + # the secret key of sqlflow user for webapi request, required true + screctKey = '' + + # sqlflow server, For the cloud version, the value is https://api.gudusoft.com + server = 'https://api.gudusoft.com' + + + # sqlflow api port, For the cloud version, the value is 80 + port = '' + + # The token is generated from userid and usersecret. It is used in every Api invocation. + token = GenerateToken.getToken(userId, server, port, screctKey) + + # sql to be checked + sql = 'select * fro1m table1' + + # database type, dbvansi,dbvathena,dbvazuresql,dbvbigquery,dbvcouchbase,dbvdb2,dbvgreenplum,dbvgaussdb,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpresto,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsparksql,dbvsybase,dbvteradata,dbvvertica + dbvendor = 'dbvoracle' + + # check syntax + check(server, port, sql, dbvendor, userId, token) diff --git a/api/python/basic/getcsv.py b/api/python/basic/getcsv.py new file mode 100644 index 0000000..a654404 --- /dev/null +++ b/api/python/basic/getcsv.py @@ -0,0 +1,105 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import requests +import json +import sys +import GenerateToken +import GenerateLineageParam + + +def getResult(server, port, data, files): + url = "/api/gspLive_backend/sqlflow/generation/sqlflow/exportFullLineageAsCsv" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/sqlflow/generation/sqlflow/exportFullLineageAsCsv' + if port != '': + url = server + ':' + port + url + else: + url = server + url + + datastr = json.dumps(data) + + print('start get csv result from sqlflow.') + try: + if files != '': + response = requests.post(url, data=eval(datastr), files=files, verify=False) + else: + response = requests.post(url, data=eval(datastr), verify=False) + except Exception as e: + print('get csv result from sqlflow failed.', e) + sys.exit(0) + + print('get csv result from sqlflow successful. result : ') + print() + return response.text + + +if __name__ == '__main__': + # the user id of sqlflow web or client, required true + userId = '' + + # the secret key of sqlflow user for webapi request, required true + screctKey = '' + + # sqlflow server, For the cloud version, the value is https://api.gudusoft.com + server = 'http://127.0.0.1' + + # sqlflow api port, For the cloud version, the value is 443 + port = '8165' + + # For the cloud version + # server = 'https://api.gudusoft.com' + # port = '80' + + # The token is generated from userid and usersecret. It is used in every Api invocation. + token = GenerateToken.getToken(userId, server, port, screctKey) + + # delimiter of the values in CSV, default would be ',' string + delimiter = ',' + + # export_include_table, string + export_include_table = '' + + # showConstantTable, boolean + showConstantTable = 'true' + + # Whether treat the arguments in COUNT function as direct Dataflow, boolean + treatArgumentsInCountFunctionAsDirectDataflow = '' + + # database type, + # dbvazuresql + # dbvbigquery + # dbvcouchbase + # dbvdb2 + # dbvgreenplum + # dbvhana + # dbvhive + # dbvimpala + # dbvinformix + # dbvmdx + # dbvmysql + # dbvnetezza + # dbvopenedge + # dbvoracle + # dbvpostgresql + # dbvredshift + # dbvsnowflake + # dbvmssql + # dbvsparksql + # dbvsybase + # dbvteradata + # dbvvertica + dbvendor = 'dbvoracle' + + # sql text + # sqltext = 'select * from table' + # data = GenerateLineageParam.buildSqltextParam(userId, token, delimiter, export_include_table, showConstantTable, treatArgumentsInCountFunctionAsDirectDataflow, dbvendor, sqltext) + # resp = getResult(server, port, data, '') + + # sql file + sqlfile = 'test.sql' + data, files = GenerateLineageParam.buildSqlfileParam(userId, token, delimiter, export_include_table, + showConstantTable, + treatArgumentsInCountFunctionAsDirectDataflow, dbvendor, + sqlfile) + resp = getResult(server, port, data, files) + print(resp) diff --git a/api/python/basic/readme.md b/api/python/basic/readme.md new file mode 100644 index 0000000..165537c --- /dev/null +++ b/api/python/basic/readme.md @@ -0,0 +1,104 @@ +## Python Data lineage: using the SQLFlow REST API (Basic) + +A basic tutorial for using the Python version of the SQLFlow API. + +Here is an advanced version of how to use [Python to get the data lineage](https://github.com/sqlparser/sqlflow_public/tree/master/api/python/advanced). + +### Prerequisites + +- Python 2.7 or higher version must be installed and configured correctly. + +- Installing Dependency Libraries: + +` +pip install requests +` + +### GenerateTokenDemo.py + +This demo shows how to get a token from a SQLFlow system that can be used to legally call other interfaces. + +* Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **userSecret**: the userSecret of sqlflow client request. sqlflow web, required false, sqlflow client, required true + +This is the user id that is used to connect to the SQLFlow server. +Always set this value to `gudu|0123456789` and keep `userSecret` empty if you use the SQLFlow on-premise version. + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + +**set the parameters in the code** + +Connect to the SQLFlow Cloud Server: + +````json + url = 'https://api.gudusoft.com/gspLive_backend/user/generateToken' + userId = 'YOUR USER ID' + screctKey = 'YOUR SECRET KEY' +```` + +Connect to the SQLFlow on-premise version: + +````json + url = 'http://127.0.0.1:8081/gspLive_backend/user/generateToken' + userId = 'gudu|012345678' + screctKey = '' +```` + +**start script** + +`python GenerateTokenDemo.py` + +### GenerateDataLineageDemo.py + +This demo shows how to get the desired SQL script analysis results from the SQLFlow system. + +* Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **userSecret**: the userSecret of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * sqltext: sql text, required false + * sqlfile: sql file, required false + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * filePath: data lineage file path + + +**set the parameters in the code** + +Connect to the SQLFlow Cloud Server: + +````json + tokenUrl = 'https://api.gudusoft.com/gspLive_backend/user/generateToken' + generateDataLineageUrl = 'https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow' + userId = 'YOUR USER ID' + screctKey = 'YOUR SECRET KEY' + sqlfile = 'test.sql' + dbvendor = 'dbvoracle' + filePath = 'datalineage' +```` + +Connect to the SQLFlow on-premise version: + +````json + tokenUrl = 'http://127.0.0.1:8081/gspLive_backend/user/generateToken' + generateDataLineageUrl = 'http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow' + userId = 'gudu|012345678' + screctKey = '' + sqlfile = 'test.sql' + dbvendor = 'dbvoracle' + filePath = 'datalineage' +```` + +**start script** + +cmd: + +- /f. the sqlfile path，required. eg: /f sql.txt +- /o. the data lineage file type. default value is json, optional. eg: /o csv , /o json + +eg: + +`python GenerateDataLineageDemo.py /f test.sql /o csv` + + diff --git a/api/python/basic/toxml.py b/api/python/basic/toxml.py new file mode 100644 index 0000000..be93aa1 --- /dev/null +++ b/api/python/basic/toxml.py @@ -0,0 +1,59 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import requests +import json +import GenerateToken + + +def toxml(server, port, sql, dbvendor, userId, token): + url = "/api/gspLive_backend/demo/xml/toXML" + if 'api.gudusoft.com' in server: + url = '/gspLive_backend/demo/xml/toXML' + if port != '': + url = server + ':' + port + url + else: + url = server + url + + data = {'sql': sql, 'dbvendor': dbvendor, 'userId': userId, 'token': token} + header_dict = {"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"} + try: + r = requests.post(url, data=data, headers=header_dict, verify=False) + except Exception as e: + print('convert failed.', e) + result = json.loads(r.text) + + usedTime = result['data']['usedTime'] + version = result['data']['gsp.version'] + if result['code'] == 200: + xml = result['data']['xml'] + print('elapsed time: ' + usedTime+' ,gsp version: ' + version + ' ,xml result: ') + print(xml) + else: + print('to xml failed. elapsed time: ' + usedTime + ' ,gsp version: ' + version + ' ,error info: ') + print(result['error']) + + +if __name__ == '__main__': + # the user id of sqlflow web or client, required true + userId = '' + + # the secret key of sqlflow user for webapi request, required true + screctKey = '' + + # sqlflow server, For the cloud version, the value is https://api.gudusoft.com + server = 'https://api.gudusoft.com' + + # sqlflow api port, For the cloud version, the value is 80 + port = '' + + # The token is generated from userid and usersecret. It is used in every Api invocation. + token = GenerateToken.getToken(userId, server, port, screctKey) + + # sql to be checked + sql = 'select * from table1' + + # database type, dbvansi,dbvathena,dbvazuresql,dbvbigquery,dbvcouchbase,dbvdb2,dbvgreenplum,dbvgaussdb,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpresto,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsparksql,dbvsybase,dbvteradata,dbvvertica + dbvendor = 'dbvoracle' + + # to xml + toxml(server, port, sql, dbvendor, userId, token) diff --git a/api/python/depreciated/README.txt b/api/python/depreciated/README.txt new file mode 100644 index 0000000..9c76b4b --- /dev/null +++ b/api/python/depreciated/README.txt @@ -0,0 +1,163 @@ + +## THIS VERSION IS DEPRECIATED, PLEASE USE THE CODE IN THE BASIC OR ADAVANCED DIRECTORY + +======================================================================================================================================================================================================== +SQLFlow API Python Client Documentation +======================================================================================================================================================================================================== + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +DESCRIPTION +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +High-level Python client of the SQLFlow API. + +SQLFlow is a product of Gudusoft. The software's purpose is to analyze the flow of data, data relationships and dependencies coded into various SQL scripts. + +This Python wrapper is built to process SQL scripts using the API with the option to export the API responses into JSON files. + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +BASIC USAGE +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +The Python client is built into a single module. To use it, one must have a valid API key (currently available for the community at https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp). + +**************************************************************************************************** + +SQLFlowClient(api_key, api_url) class stores relevant parameters and methods to utilize SQLFlow API. + +It has all the default values included for both the API key (which is currently available to the public) and the API base URL. + +Initializig it will create an object with the following variables: API key, API URL, and it will also initialize the default request header and a default API parameter configuration. + +**************************************************************************************************** + +configure_api(db_vendor, rel_type, simple_output, ignore_rs) method is created to change default API parameters as per required. It will change the pre-set API configuration based to provided parameter values. + +Detailed explanations regarding API configuration could be found here: https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp and here: https://api.gudusoft.com/gspLive_backend/swagger-ui.html#!/sqlflow-controller/generateSqlflowUsingPOST. + +While using the method, one must provide all four parameters. Omitting one will result in error, while passing an invalid value will result in a notification and both will prevent the client from configuring the API request, and a notification message will be returned. + +Valid parameters are as follows: + +- db_vendor: dbvbigquery, dbvcouchbase, dbvdb2, dbvgreenplum, dbvhana, dbvhive, dbvimpala, dbvinformix, dbvmdx, dbvmysql, dbvnetezza, dbvopenedge, dbvoracle, dbvpostgresql, dbvredshift, dbvsnowflake, dbvmssql, bvsybase, dbvteradata, dbvvertica + +- rel_type: fdd, fdr, frd, fddi, join + +- simple_output: true, false + +- ignore_rs: true, false + +**************************************************************************************************** + +analyze_script(script_path) method can be used to submit a SQL script to the SQLFlow API for analysis. If the analysis returns a response sucessfully, the results will be stored in the SQLFlowClient object's results variable. Results variable is a dictionary object containing script paths and API responses as key-value pairs. + +The method won't perform if the in-built check of the provided file path is not pointing to a SQL script. This will result in a notification message instead. + +If the API call results in an error (e.g. invalid API key, server being busy), the response won't be stored, but a notification message will be returned instead. + +**************************************************************************************************** + +export_results(export_folder) method simply dumps all the API call results stored already in SQLFlowClient's results variable to the specified output folder path. + +The API responses will be saved as JSON files, with a filename corresponding to their source scripts'. + +If the provided path doesn't exist, the method will automatically build the path. + +If there are no stored responses yet, the function won't perform, and will return a notification message. + +**************************************************************************************************** + +mass_process_scripts(source_folder, export_folder = None) method will scan the entire directory tree of the provided source folder for SQL script files and submits each to the API, storing all the responses in the results variable. + +It can optionally export the results of the detected scripts to a desired export folder. If export_folder is left as None, this operation will be skipped. + +Please note that this method will only execute the exporting of API results of scripts which were discovered in the specified directory at the function's execution. + +**************************************************************************************************** + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +CODE EXAMPLES +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +# Initialize API client + +client = SQLFlowClient() + +# ============================================================================= + +# Configure the API parameters + +client.configure('dbvmssql', 'fddi', 'false', 'false') + +# Check config values after setting the parameters + +print(client.config) + +# ============================================================================= + +# Execute the analysis of a single script file + +client.analyze_script('C:/Users/TESTUSER/Desktop/EXAMPLESCRIPT.sql') + +# Check stored API response of the previous step + +print(client.results) + +# ============================================================================= + +# Export the stored response + +client.export_results('C:/Users/TESTUSER/Desktop/EXPORTFOLDER') + +# ============================================================================= + +# Execute mass processing of SQL scripts in a folder with an export folder specified + +client.mass_process_scripts('C:/Users/TESTUSER/Desktop/SOURCEFOLDER', 'C:/Users/TESTUSER/Desktop/EXPORTFOLDER') + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +AUTHORS +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +Bence Kiss (vencentinus@gmail.com) + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +ADDITIONAL INFORMATION +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +Detailed information about the SQLFlow project could be accessed via the following links: + +API configuration https://api.gudusoft.com/gspLive_backend/swagger-ui.html#!/sqlflow-controller/generateSqlflowUsingPOST + +SQLFlow Git repo https://github.com/sqlparser/sqlflow_public + +Dataflow relationship types https://github.com/sqlparser/sqlflow_public/blob/master/dbobjects_relationship.md + +SQLFlow front end http://www.gudusoft.com/sqlflow/#/ + +C# API client https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp + + +In case of any questions regarding SQLFlow please contact Mr. James Wang at info@sqlparser.com. + +In case of bugs, comments, questions etc. please feel free to contact the author at vencentinus@gmail.com or Mr. James Wang at info@sqlparser.com. + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +ACKNOWLEDGEMENTS +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +The author of this project acknowledges that SQLFlow is a product and intellectual property exclusively of Gudusoft. + +This project has been created to facilitate the utilization of the tool by the community, and the author of this Python client neither received nor expects to receive any compensation from Gudusoft in exchange. + +This development has been created with good faith and with the intention to contribute to a great development, which the author of this wrapper has been utilizing for free under its development period. + +The code is free to use for anyone intending to use SQLFlow API in any manner. + +Thanks to Mr. James Wang, CTO of Gudusoft for his kind support and allowing me to utilize the tool under it's development and contribute to his company's project. \ No newline at end of file diff --git a/api/python/depreciated/SQLFlow_API_Python_Client.py b/api/python/depreciated/SQLFlow_API_Python_Client.py new file mode 100644 index 0000000..9dc23bc --- /dev/null +++ b/api/python/depreciated/SQLFlow_API_Python_Client.py @@ -0,0 +1,371 @@ +''' +************************************************************************************************************************************************************ + +Properties +================ +NAME: SQLFlow API Python Client +DESCRIPTION: A simple wrapper written for Gudusoft's SQLFlow API. +AUTHOR: Bence Kiss +ORIGIN DATE: 21-MAR-2020 +PYTHON VERSION: 3.7.3 + +Additional Notes +================ +- + + +ADDITIONAL INFORMATION +============================================================================================================================================================ +Resources URL +============================== ============================================================================================================================ +API configuration https://api.gudusoft.com/gspLive_backend/swagger-ui.html#!/sqlflow-controller/generateSqlflowUsingPOST +------------------------------ ---------------------------------------------------------------------------------------------------------------------------- +SQLFlow Git repo https://github.com/sqlparser/sqlflow_public +------------------------------ ---------------------------------------------------------------------------------------------------------------------------- +Dataflow relationship types https://github.com/sqlparser/sqlflow_public/blob/master/dbobjects_relationship.md +------------------------------ ---------------------------------------------------------------------------------------------------------------------------- +SQLFlow front end http://www.gudusoft.com/sqlflow/#/ +------------------------------ ---------------------------------------------------------------------------------------------------------------------------- +C# API client https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp +------------------------------ ---------------------------------------------------------------------------------------------------------------------------- + + +REVISION HISTORY +============================================================================================================================================================ +Version Change Date Author Narrative +======= =============== ====== ============================================================================================================================ +1.0.0 21-MAR-2020 BK Created +------- --------------- ------ ---------------------------------------------------------------------------------------------------------------------------- +0.0.0 DD-MMM-YYYY XXX What changed and why... +------- --------------- ------ ---------------------------------------------------------------------------------------------------------------------------- + +************************************************************************************************************************************************************ +''' + +# ========================================================================================================================================================== + +# Import required modules + +import os +import requests +import json + +# ========================================================================================================================================================== + +class SQLFlowClient: + + ''' + + Class description + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + + Class containing various functions to use SQLFlow API. + + Class instance variables + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + + - api_key: The token needed for authorization. Default public token can be found here: + + https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp + + - api_url: Default base URL of the API requests. Can be changed at class initialization. + + Class methods + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + + - configure_api: Set the API parameters for the requests. + - analyze_script: Submit a single SQL script using POST request to the API. Responses are stored in the class instance's results variable. + - export_responses: Export all stored API responses to a target folder as JSON files. + - mass_process_scripts: Process all SQL scripts found in a directory tree, optionally exporting results to a designated folder. + + Class dependencies + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + + Packages used in the script considered to be core Python packages. + + - os: Used to handle input/output file and folder paths. + - requests: Used to generate POST requests and submit script files to the API. + - json: Used to process API responses when it comes to exporting. + + ''' + + # ========================================================================================================================================================== + # ========================================================================================================================================================== + + def __init__(self, + api_key = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYwMzc1NjgwMCwiaWF0IjoxNTcyMjIwODAwfQ.EhlnJO7oqAHdr0_bunhtrN-TgaGbARKvTh2URTxu9iU', + api_url = 'https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow' + ): + + ''' + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + Initialize SQLFlow API client. + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + ''' + + # Set instance variables + + self.key = api_key + + self.url = api_url + + # Set default request header + + self.headers = {'Accept': 'application/json;charset=utf-8', + 'Authorization': self.key + } + + # ============================================================================= + + # Set lists of allowed API configuration values + + # List of allowed database vendors + + self.dbvendors = ['dbvbigquery', + 'dbvcouchbase', + 'dbvdb2', + 'dbvgreenplum', + 'dbvhana', + 'dbvhive', + 'dbvimpala', + 'dbvinformix', + 'dbvmdx', + 'dbvmysql', + 'dbvnetezza', + 'dbvopenedge', + 'dbvoracle', + 'dbvpostgresql', + 'dbvredshift', + 'dbvsnowflake', + 'dbvmssql', + 'dbvsybase', + 'dbvteradata', + 'dbvvertica' + ] + + # List of allowed data relationship types + + self.reltypes = ['fdd', + 'fdr', + 'frd', + 'fddi', + 'join' + ] + + # List of allowed values for Boolean parameters + + self.switches = ['true', + 'false' + ] + + # ============================================================================= + + # Set default API configuration + + self.config = {'dbvendor': 'dbvmssql', + 'showRelationType': 'fdd', + 'simpleOutput': 'false', + 'ignoreRecordSet': 'false' + } + + # Variable to store API responses + + self.results = dict() + + # ========================================================================================================================================================== + # ========================================================================================================================================================== + + def configure_api(self, + db_vendor, + rel_type, + simple_output, + ignore_rs + ): + + ''' + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + Configure the API request parameters. Only works if all parameters are provided. + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + ''' + + # Check if the provided configuration values are valid + + if db_vendor in self.dbvendors and rel_type in self.reltypes and simple_output in self.switches and ignore_rs in self.switches: + + # Assign valid configuration parameters to config variable + + self.config = {'dbvendor': db_vendor, + 'showRelationType': rel_type, + 'simpleOutput': simple_output, + 'ignoreRecordSet': ignore_rs + } + + # If any of the provided parameters are invalid, quit function and notify user + + else: + + print('\n\n' + '=' * 75 + '\n\nOne or more configuration values are missing or invalid. Please try again.\n\nAllowed values for db_vendor:\n\n' + + ' / '.join(self.dbvendors) + + '\n\nAllowed values for relation_type:\n\n' + + ' / '.join(self.reltypes) + + '\n\nAllowed values for simple_output and ignore_rs:\n\n' + + ' / '.join(self.switches) + + '\n\n' + '=' * 75 + ) + + # ========================================================================================================================================================== + # ========================================================================================================================================================== + + def analyze_script(self, + script_path + ): + + ''' + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + Submit SQL script file for SQLFlow analysis. + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + ''' + + # Compile the API request URL + + configuredURL = self.url + '?' + ''.join(str(parameter) + '=' + str(setting) + '&' for parameter, setting in self.config.items()).rstrip('&') + + # ============================================================================= + + # Check if provided path points to a SQL script file + + if os.path.isfile(script_path) and script_path.lower().endswith('.sql'): + + # Open the script file in binary mode so it could be submitted in a POST request + + with open(script_path, mode = 'rb') as scriptFile: + + # Use requests module's POST function to submit file and retrieve API response + + response = requests.post(configuredURL, files = {'sqlfile': scriptFile}, headers = self.headers) + + # ============================================================================= + + # Add the request response to the class variable if response was OK + + if response.status_code == 200: + + self.results[script_path] = json.loads(response.text) + + # If response returned a different status, quit function and notify user + + else: + + print('\nAn invalid response was returned for < ' + os.path.basename(script_path) + ' >.\n', '\nStatus code: ' + str(response.status_code) + '\n') + + # If script file's path is invalid, quit function and notify user + + else: + + print('\nProvided path is not pointing to a SQL script file. Please try again.\n') + + # ========================================================================================================================================================== + # ========================================================================================================================================================== + + def export_results(self, + export_folder + ): + + ''' + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + Export all stored API responses as JSON files to a specified folder. + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + ''' + + # Check if there are responses to be exported + + if len(self.results) != 0: + + # Create the directory for the result files if it doesn't exist + + os.makedirs(export_folder, exist_ok = True) + + # ============================================================================= + + # Iterate the API results stored in the class + + for scriptpath, response in self.results.items(): + + # Create a JSON file and export API results of each processed script file into the JSON file + + with open(os.path.join(export_folder, os.path.basename(scriptpath).replace('.sql', '') + '.json'), mode = 'w') as resultFile: + + # Write the response into the JSON file + + json.dump(response, resultFile) + + # If there are no responses yet, quit function and notify user + + else: + + print('\nThere are no API responses stored by the client yet.\n') + + # ========================================================================================================================================================== + # ========================================================================================================================================================== + + def mass_process_scripts(self, + source_folder, + export_folder = None): + + ''' + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + Scan a directory tree for SQL script files and pass each to an API call. Optionally export results to a desired folder. + ------------------------------------------------------------------------------------------------------------------------------------------------------------ + ''' + + # List to store SQL script file paths found in source folder + + scriptPaths = list() + + # ============================================================================= + + # Scan source folder and subfolders + + for (dirpath, dirnames, filenames) in os.walk(source_folder): + + # Collect all paths which refer SQL scripts + + scriptPaths += [os.path.join(dirpath, file) for file in filenames if os.path.isfile(os.path.join(dirpath, file)) and file.lower().endswith('.sql')] + + # ============================================================================= + + # If there is at least one SQL script in the directory tree execute API call + + if len(scriptPaths) != 0: + + # Iterate the SQL scrip paths and call the API for each file + + [self.analyze_script(script_path = path) for path in scriptPaths] + + # ============================================================================= + + # If an export folder is provided, save the responses to that folder (but only those which have been analyzed at function call) + + if export_folder: + + # Store the current set of API responses + + allResults = self.results + + # Filter for responses related to current function call + + self.results = {scriptpath: response for scriptpath, response in self.results.items() if scriptpath in scriptPaths} + + # Export the responses of the current function call to the desired target folder + + self.export_results(export_folder = export_folder) + + # Reset the results variable to contain all responses again + + self.results = allResults + + # If no SQL script files were found in the directory tree, quit finction and notify user + + else: + + print('\nNo SQL script files have been found in the specified source folder and its subfolders.\n') \ No newline at end of file diff --git a/api/readme.md b/api/readme.md new file mode 100644 index 0000000..4af9b28 --- /dev/null +++ b/api/readme.md @@ -0,0 +1,121 @@ +## How to use the Rest API of SQLFlow + +This article describes how to use the Rest API provided by the SQLFlow to +communicate with the SQLFlow server and get the generated metadata and data lineage. + +In this article, we use `Curl` to demonstrate the usage of the Rest API, +you can use any preferred programming language as you like. + +### Prerequisites + +In order to use the SQLFlow rest API, you may connect to the [**SQLFlow Cloud server**](https://sqlflow.gudusoft.com), +Or, setup a [**SQLFlow on-premise version**](https://www.gudusoft.com/sqlflow-on-premise-version/) on your owner server. + + +1. **SQLFlow Cloud server** + +- User ID +- Secrete Key + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + + +2. **SQLFlow on-premise version** + +Please [check here](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow.md) to see how to install SQLFlow on-premise version on you own server. + +- User ID +- Secrete Key + +Always set userId to `gudu|0123456789` and keep `userSecret` empty when connect to the SQLFlow on-premise version. + + +### Difference of the API calls between SQLFlow Cloud server and SQLFlow on-premise version + +1. TOKEN is not needed in the API calls when connect to the SQLFlow on-premise version +2. userId alwyas set to `gudu|0123456789` and `userSecret` leave empty when connect to the SQLFlow on-premise version. +3. The server port is 8081 by default for the SQLFlow on-premise version, and There is no need to specify the port when connect to the SQLFlow Cloud server. + +Regarding the server port of the SQLFlow on-premise version, please [check here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#1-sqlflow-server) for more information. + + + +### Using the Rest API + +#### 1. Generate a token + + +Once you have the `userid` and `secret key`, the first API need to call is: + +``` +/gspLive_backend/user/generateToken +``` + +This API will return a temporary token that needs to be used in the API call thereafter. + +**SQLFlow Cloud Server** +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/user/generateToken" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:application/x-www-form-urlencoded;charset=UTF-8" -d "secretKey=YOUR SECRET KEY" -d "userId=YOUR USER ID HERE" +``` + +**SQLFlow on-premise version** + +TOKEN is not needed in the on-premise version. So, there is no need to generate a token. + + +#### 2. Generate the data lineage + +Call this API by sending the SQL query and get the result includes the data lineage. + +``` +/gspLive_backend/sqlflow/generation/sqlflow +``` + + +**SQLFlow Cloud Server** +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow?showRelationType=fdd" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "sqlfile=" -F "dbvendor=dbvoracle" -F "ignoreRecordSet=false" -F "simpleOutput=false" -F "sqltext=CREATE VIEW vsal as select * from emp" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" +``` + +**SQLFlow on-premise version** +``` +curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow?showRelationType=fdd" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "sqlfile=" -F "dbvendor=dbvoracle" -F "ignoreRecordSet=false" -F "simpleOutput=false" -F "sqltext=CREATE VIEW vsal as select * from emp" -F "userId=gudu|0123456789" +``` + + +#### 3. Export the data lineage in csv format + +Call this API by sending the SQL file and get the csv result includes the data lineage. + +``` +/gspLive_backend/sqlflow/generation/sqlflow/exportLineageAsCsv +``` + +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow/exportLineageAsCsv" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "dbvendor=dbvoracle" -F "showRelationType=fdd" -F "sqlfile=@YOUR UPLOAD FILE PATH HERE" --output YOUR DOWNLOAD FILE PATH HERE +``` + +Sample: +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow/exportLineageAsCsv" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=auth0|5fc8e95991a780006f180d4d" -F "token=YOUR TOKEN HERE" -F "dbvendor=dbvoracle" -F "showRelationType=fdd" -F "sqlfile=@c:\prg\tmp\demo.sql" --output c:\prg\tmp\demo.csv +``` + + +**Note:** + * -H "Content-Type:multipart/form-data" is required. + * Add **@** before the upload file path + * --output is required. + * Optional, if you just want to fetch table to table relations, please add **-F "tableToTable=true"** + + +#### 4. Submit multiple SQL files and get the data lineage in CSV, JSON, graphml format. +Rest APIs: Job + +### The full reference to the Rest APIs + +[SQLFlow rest API reference](sqlflow_api.md) + +### Troubleshooting + +- Under windows, you may need to add option `--ssl-no-revoke` to avoid some security issues, `curl --ssl-no-revoke` diff --git a/api/sqlflow-job-api-tutorial.md b/api/sqlflow-job-api-tutorial.md new file mode 100644 index 0000000..21a66dc --- /dev/null +++ b/api/sqlflow-job-api-tutorial.md @@ -0,0 +1,255 @@ +- [SQLFlow Job API tutorial](#sqlflow-job-api-tutorial) + * [1. Prerequisites](#1-prerequisites) + + [Difference of the API calls between SQLFlow Cloud server and SQLFlow on-premise version](#difference-of-the-api-calls-between-sqlflow-cloud-server-and-sqlflow-on-premise-version) + + [Generate a token](#generate-a-token) + * [2. Different type of Job](#2-different-type-of-job) + * [3. Simple job rest API](#3-simple-job-rest-api) + + [1. Submit a sqlflow job](#1-submit-a-sqlflow-job) + + [2. Get job status](#2-get-job-status) + + [3. Export data lineage](#3-export-data-lineage) + * [4. Regular job rest API](#4-regular-job-rest-api) + + +## SQLFlow Job API tutorial + +This article describes how to use the Job Rest API provided by the SQLFlow to +communicate with the SQLFlow server and export the data lineage in json, csv, graphml formats. + +### 1. Prerequisites +In order to use the SQLFlow rest API, you may connect to the [**SQLFlow Cloud server**](https://sqlflow.gudusoft.com), +Or, setup a [**SQLFlow on-premise version**](https://www.gudusoft.com/sqlflow-on-premise-version/) on your owner server. + + +1. **SQLFlow Cloud server** + +- User ID +- Secrete Key + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + + +2. **SQLFlow on-premise version** + +Please [check here](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow.md) to see how to install SQLFlow on-premise version on you own server. + +- User ID +- Secrete Key + +Always set userId to `gudu|0123456789` and keep `userSecret` empty when connect to the SQLFlow on-premise version. + + +#### Difference of the API calls between SQLFlow Cloud server and SQLFlow on-premise version + +1. TOKEN is not needed in the API calls when connect to the SQLFlow on-premise version +2. userId alwyas set to `gudu|0123456789` and `userSecret` leave empty when connect to the SQLFlow on-premise version. +3. The server port is 8081 by default for the SQLFlow on-premise version, and There is no need to specify the port when connect to the SQLFlow Cloud server. + +Regarding the server port of the SQLFlow on-premise version, please [check here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#1-sqlflow-server) for more information. + +#### Generate a token + +Once you have the `userid` and `secret key`, the first API need to call is: + +``` +/gspLive_backend/user/generateToken +``` + +This API will return a temporary token that needs to be used in the API call thereafter. + +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/user/generateToken" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:application/x-www-form-urlencoded;charset=UTF-8" -d "secretKey=YOUR SECRET KEY" -d "userId=YOUR USER ID HERE" +``` + +More detail, please see https://github.com/sqlparser/sqlflow_public/edit/master/api/readme.md + +### 2. Different type of Job +![SQLFlow job types](job-types.png) + +### 3. Simple job rest API + +#### 1. Submit a sqlflow job + +Call this API by sending the SQL files and get the result includes the data lineage. SQLFlow job supports both of multiple files and zip archive file. + +``` +/gspLive_backend/sqlflow/job/submitUserJob +``` + +Example in `Curl` +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/submitUserJob" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "sqlfiles=@FIRST FILE PATH" -F "sqlfiles=@SECOND FILE PATH" -F "dbvendor=dbvmssql" -F "jobName=job1" +``` + +**Note:** + * **-H "Content-Type:multipart/form-data"** is required + * Add **@** before the file path + +Return data: +```json +{ + "code":200, + "data":{ + "jobId":"c359aef4bd9641d697732422debd8055", + "jobName":"job1", + "userId":"google-oauth2|104002923119102769706", + "dbVendor":"dbvmssql", + "dataSource":{ + + }, + "fileNames":["1.sql","1.zip"], + "createTime":"2020-12-15 15:14:39", + "status":"create" + } +} +``` + +Please records the jobId field. + +#### 2. Get job status + + * Get the specify user job status and summary + + ``` + /gspLive_backend/sqlflow/job/displayUserJobSummary + ``` + + Example in `Curl` + + ```json + curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/displayUserJobSummary" -F "jobId=c359aef4bd9641d697732422debd8055" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" + ``` + + Return data: + ```json + { + "code":200, + "data":{ + "jobId":"c359aef4bd9641d697732422debd8055", + "jobName":"job1", + "userId":"google-oauth2|104002923119102769706", + "dbVendor":"dbvmssql", + "dataSource":{ + + }, + "fileNames":["1.sql","1.zip"], + "createTime":"2020-12-15 15:14:39", + "status":"success", + "sessionId":"fe5898d4e1b1a7782352b50a8203ca24c04f5513446e9fb059fc4d584fab4dbf_1608045280033" + } + } + ``` + + * Get all jobs (include history jobs) status and summary + + ``` + /gspLive_backend/sqlflow/job/displayUserJobsSummary + ``` + + Example in `Curl` + + ```json + curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/displayUserJobsSummary" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" + ``` + + + +#### 3. Export data lineage + + When the job status is **success**, you can export the data lineage in json, csv, graphml formats + + * 3.1 Export data lineage in json format + + ``` + /gspLive_backend/sqlflow/job/exportLineageAsJson + ``` + + Example in `Curl` + + ``` + curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/exportLineageAsJson" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "jobId=c359aef4bd9641d697732422debd8055" --output lineage.json + ``` + **Note:** + > If you want to get table to table relation, please add option -F "tableToTable=true" + + * 3.2 Export data lineage in csv format + + ``` + /gspLive_backend/sqlflow/job/exportFullLineageAsCsv + ``` + + Example in `Curl` + + ``` + curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/exportFullLineageAsCsv" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "jobId=c359aef4bd9641d697732422debd8055" --output lineage.csv + ``` + + **Note:** + > If you want to get table to table relation, please add option -F "tableToTable=true" + + > If you want to change csv delimiter, please add option -F "delimiter=<delimiter char>" + + + * 3.3 Export data lineage in graphml format, you can view the lineage graph at yEd Graph Editor + + ``` + /gspLive_backend/sqlflow/job/exportLineageAsGraphml + ``` + + Example in `Curl` + + ``` + curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/exportLineageAsGraphml" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "jobId=c359aef4bd9641d697732422debd8055" --output lineage.graphml + ``` + + **Note:** + > If you want to get table to table relation, please add option -F "tableToTable=true" + +### 4. Regular job rest API + +#### 1. Submit a regular job + +Call this API by sending the SQL files and get the result includes the data lineage. SQLFlow job supports both of multiple files and zip archive file. + +If the job is incremental, please set incremental=true + * first submit, jobId is null, and record the jobId field from response message + * second submit, jobId can't be null, please fill the jobId which returns by the first submit response. + +``` +/gspLive_backend/sqlflow/job/submitPersistJob +``` + +Example in `Curl` +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/submitPersistJob" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "sqlfiles=@FIRST FILE PATH" -F "sqlfiles=@SECOND FILE PATH" -F "dbvendor=dbvmssql" -F "jobName=job1" -F "incremental=true" +``` + +Incremental submit in `Curl` +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/job/submitPersistJob" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" -F "sqlfiles=@FIRST FILE PATH" -F "sqlfiles=@SECOND FILE PATH" -F "dbvendor=dbvmssql" -F "jobName=job1" -F "incremental=true" -F "jobId=JobId OF FIRST SUBMIT" +``` + +**Note:** + * **-H "Content-Type:multipart/form-data"** is required + * Add **@** before the file path + +Return data: +```json +{ + "code":200, + "data":{ + "jobId":"c359aef4bd9641d697732422debd8055", + "jobName":"job1", + "userId":"google-oauth2|104002923119102769706", + "dbVendor":"dbvmssql", + "dataSource":{ + + }, + "fileNames":["1.sql","1.zip"], + "createTime":"2020-12-15 15:14:39", + "status":"create" + } +} +``` + +Please records the jobId field. diff --git a/api/sqlflow_api.md b/api/sqlflow_api.md new file mode 100644 index 0000000..a12d4e2 --- /dev/null +++ b/api/sqlflow_api.md @@ -0,0 +1,465 @@ +# SQLFlow WebAPI + + +## JWT Client API Authorization (for sqlflow client api call) +* All of the restful requests are based on JWT authorization. Before accessing the sqlflow WebAPI, client user needs to obtain the corresponding JWT token for legal access. + +* How to get JWT Token + 1. Login on [the sqlflow web](https://sqlflow.gudusoft.com), upgrade to premium account. + 2. Move mouse on the login user image, select Account menu item, click the "generate" button to generate the user secret key. + 3. When you get the user secret key, you can call **/gspLive_backend/user/generateToken** api to obtain a token, the ttl of new token is 24 hours. + 4. **/gspLive_backend/user/generateToken** + * **userId**: the user id of sqlflow web or client, required **true** + * **secretKey**: the secret key of sqlflow user for webapi request, required **true** + +* How to use JWT Token for security authentication? + * Each webapi contains two parameters, named userId and token. + +## WebAPI + +### Sqlflow Generation Interface + +* **/sqlflow/generation/sqlflow/graph** + * Description: generate sqlflow model and graph + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: The token is only used when connecting to the SQLFlow Cloud server, not used when connect to the SQLFlow on-premise version. + * sqltext: sql text, optional + * sqlfile: sql file, optional + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * simpleOutput: simple output, ignore the intermediate results, defualt is false. + * ignoreRecordSet: same as simpleOutput, but will keep output of the top level select list, default is false. + * dataflowOfAggregateFunction, treat the dataflow generated by the aggregate function as direct dataflow or not ,default is direct. + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + * showConstantTable: return constant or not, default is false. + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * showRelationType: show relation type, optional, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **join**: combine rows from two or more tables, based on a related column between them + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + + + * Sample: + * test sql: + ```sql + select name from user + ``` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/graph" -H "accept:application/json;charset=utf-8" -F "userId=your user id here" -F "token=your token here" -F "dbvendor=dbvoracle" -F "ignoreFunction=true" -F "ignoreRecordSet=true" -F "sqltext=select name from user" + ``` + * response: + ```json + { + "code": 200, + "data": { + "mode": "global", + "summary": { + ... + }, + "sqlflow": { + "dbvendor": "dbvoracle", + "dbobjs": [ + ... + ] + }, + "graph": { + "elements": { + "tables": [ + ... + ], + "edges": [ + ... + ] + }, + "tooltip": {}, + "relationIdMap": { + ... + }, + "listIdMap": { + ... + } + } + }, + "sessionId": "6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + } + ``` + +* **/sqlflow/generation/sqlflow/selectedgraph** + * Description: generate sqlflow model and selected dbobject graph + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **sessionId**: request sessionId, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * database: selected database, required false + * schema: selected schema, required false + * table: selected table, required false + * isReturnModel: whether return the sqlflow model, required false, default value is true + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether output relation simply, required false, default value is false + * ignoreRecordSet: whether ignore the record sets, required false, default value is false + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * session id: `6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/selectedgraph" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "dbvendor=dbvoracle" -F "ignoreFunction=true" -F "ignoreRecordSet=true" -F "isReturnModel=false" -F "sessionId=6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" -F "table=user" + + ``` + * response: + ```json + { + "code": 200, + "data": { + "mode": "global", + "summary": { + ... + }, + "graph": { + "elements": { + "tables": [ + ... + ], + "edges": [ + ... + ] + }, + "tooltip": {}, + "relationIdMap": { + ... + }, + "listIdMap": { + ... + } + } + }, + "sessionId": "6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + } + ``` +* **/sqlflow/generation/sqlflow/getSelectedDbObjectInfo** + * Description: get the selected dbobject information, such as file information, sql index, dbobject positions, sql which contains selected dbobject. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **sessionId**: request sessionId, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * **coordinates**: the select dbobject positions, it's a json array string, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * session id: `6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051` + * coordinates: `[{'x':1,'y':8,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'},{'x':1,'y':12,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'}]` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/getSelectedDbObjectInfo" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "coordinates=[{'x':1,'y':8,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'},{'x':1,'y':12,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'}]" -F "sessionId=6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + ``` + * response: + ```json + { + "code": 200, + "data": [ + { + "index": 0, + "positions": [ + { + "x": 1, + "y": 8 + }, + { + "x": 1, + "y": 12 + } + ], + "sql": "select name from user" + } + ] + } + ``` + +### Sqlflow User Job Interface +* **/sqlflow/job/submitUserJob** + * Description: submit user job for multiple sql files, support zip file. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobName**: job name, required **true** + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * **sqlfiles**: request sql files, please use **multiple parts** to submit the sql files, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * curl command: **Note: please add **@** before the sql file path** + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/submitUserJob" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "sqlfiles=@D:/sql.txt" -F "dbvendor=dbvoracle" -F "jobName=job_test" + ``` + * response: + ```json + { + "code": 200, + "data": { + "jobId": "6218721f092540c5a771ca8f82986be7", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "defaultDatabase": "", + "defaultSchema": "", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:11:28", + "status": "create" + } + } + ``` + +* **/sqlflow/job/displayUserJobsSummary** + * Description: get the user jobs summary information. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/displayUserJobsSummary" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" + ``` + * response: + ```json + { + "code": 200, + "data": { + "total": 1, + "success": 1, + "partialSuccess": 0, + "fail": 0, + "jobIds": [ + "bb996c1ee5b741c5b4ff6c2c66c371dd" + ], + "jobDetails": [ + { + "jobId": "bb996c1ee5b741c5b4ff6c2c66c371dd", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:16:11", + "status": "success", + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + ] + } + } + ``` + +* **/sqlflow/job/displayUserJobSummary** + * Description: get the specify user job information. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user jobs summary detail, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/displayUserJobSummary" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" + ``` + * response: + ```json + { + "code": 200, + "data": { + "total": 1, + "success": 1, + "partialSuccess": 0, + "fail": 0, + "jobIds": [ + "bb996c1ee5b741c5b4ff6c2c66c371dd" + ], + "jobDetails": [ + { + "jobId": "bb996c1ee5b741c5b4ff6c2c66c371dd", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:16:11", + "status": "success", + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + ] + } + } + ``` + +* **/sqlflow/job/deleteUserJob** + * Description: delete the user job by job id. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user job detail, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/deleteUserJob" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" + ``` + * response: + ```json + { + "code": 200, + "data": { + "jobId": "bb996c1ee5b741c5b4ff6c2c66c371dd", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:16:11", + "status": "delete", + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + } + ``` + +* **/sqlflow/job/displayUserJobGraph** + * Description: get the sqlflow job's model and graph + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user jobs summary detail, required **true** + * database: selected database, required false + * schema: selected schema, required false + * table: selected table, required false + * isReturnModel: whether return the sqlflow model, required false, default value is true + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether output relation simply, required false, default value is false + * ignoreRecordSet: whether ignore the record sets, required false, default value is false + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/displayUserJobGraph?showRelationType=fdd&showRelationType=" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" -F "ignoreFunction=true" -F "ignoreRecordSet=true" -F "isReturnModel=false" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" -F "table=user" + ``` + * response: + ```json + { + "code": 200, + "data": { + "mode": "global", + "summary": { + ... + }, + "graph": { + "elements": { + "tables": [ + ... + ], + "edges": [ + ... + ], + }, + "tooltip": {}, + "relationIdMap": { + ... + }, + "listIdMap": { + ... + } + } + }, + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + ``` + +* **/sqlflow/job/updateUserJobGraphCache** + * Description: update the user job graph cache, then user can call **/sqlflow/generation/sqlflow/selectedgraph** by sessionId, the sessionId value is from job detail. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user job detail, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/updateUserJobGraphCache" -H "Request-Origion:SwaggerBootstrapUi" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" + ``` + * response: + ```json + { + "code": 200, + "data": { + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + } + ``` +## Swagger +More information, please check the test environment swagger document: + + * http://111.229.12.71:8081/gspLive_backend/doc.html?lang=en + * Token: `eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYwMzc1NjgwMCwiaWF0IjoxNTcyMjIwODAwfQ.EhlnJO7oqAHdr0_bunhtrN-TgaGbARKvTh2URTxu9iU` diff --git a/api/sqlflow_api_full.md b/api/sqlflow_api_full.md new file mode 100644 index 0000000..7f38e1a --- /dev/null +++ b/api/sqlflow_api_full.md @@ -0,0 +1,517 @@ +# SQLFlow WebAPI + +## JWT WEB Authorization (Only for sqlflow web) +* All of the restful requests are based on JWT authorization. Before accessing the sqlflow WebAPI, web user needs to obtain the corresponding JWT token for legal access. +* How to use JWT Token for security authentication? + * In the header of the HTTP request, please pass the parameters: + ``` + Key: Authorization + Value: Token + ``` + +## JWT Client API Authorization (for sqlflow client api call) +* All of the restful requests are based on JWT authorization. Before accessing the sqlflow WebAPI, client user needs to obtain the corresponding JWT token for legal access. + +* How to get JWT Token + 1. Login on the sqlflow web + 2. Move mouse on the login user image, click the "generate token" menu item, you can get the user secret key and token, the ttl of token is 24 hours. + 3. When you get the user secret key, you can call **/gspLive_backend/user/generateToken** api to refresh token, the ttl of new token is 24 hours. + 4. **/gspLive_backend/user/generateToken** + * **userId**: the user id of sqlflow web or client, required **true** + * **secretKey**: the secret key of sqlflow user for webapi request, required **true** + +* How to use JWT Token for security authentication? + * Each webapi contains two parameters, named userId and token. + +## WebAPI + +### Sqlflow Generation Interface + +* **/sqlflow/generation/sqlflow** + * Description: generate sqlflow model + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * sqltext: sql text, required false + * sqlfile: sql file, required false + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether simple output relation, required false, default value is false + * ignoreRecordSet: whether ignore the record set, required false, default value is false + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "dbvendor=dbvoracle" -F "ignoreRecordSet=true" -F "sqltext=select name from user" + ``` + * response: + ```json + { + "code": 200, + "data": { + "dbvendor": "dbvoracle", + "dbobjs": [ + ... + ], + "relations": [ + ... + ] + }, + "sessionId": "6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501108040" + } + ``` + +* **/sqlflow/generation/sqlflow/graph** + * Description: generate sqlflow model and graph + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * sqltext: sql text, required false + * sqlfile: sql file, required false + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether output relation simply, required false, default value is false + * ignoreRecordSet: whether ignore the record sets, required false, default value is false + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/graph" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "dbvendor=dbvoracle" -F "ignoreFunction=true" -F "ignoreRecordSet=true" -F "sqltext=select name from user" + ``` + * response: + ```json + { + "code": 200, + "data": { + "mode": "global", + "summary": { + ... + }, + "sqlflow": { + "dbvendor": "dbvoracle", + "dbobjs": [ + ... + ] + }, + "graph": { + "elements": { + "tables": [ + ... + ], + "edges": [ + ... + ] + }, + "tooltip": {}, + "relationIdMap": { + ... + }, + "listIdMap": { + ... + } + } + }, + "sessionId": "6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + } + ``` + +* **/sqlflow/generation/sqlflow/selectedgraph** + * Description: generate sqlflow model and selected dbobject graph + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **sessionId**: request sessionId, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * database: selected database, required false + * schema: selected schema, required false + * table: selected table, required false + * isReturnModel: whether return the sqlflow model, required false, default value is true + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether output relation simply, required false, default value is false + * ignoreRecordSet: whether ignore the record sets, required false, default value is false + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * session id: `6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/selectedgraph" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "dbvendor=dbvoracle" -F "ignoreFunction=true" -F "ignoreRecordSet=true" -F "isReturnModel=false" -F "sessionId=6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" -F "table=user" + + ``` + * response: + ```json + { + "code": 200, + "data": { + "mode": "global", + "summary": { + ... + }, + "graph": { + "elements": { + "tables": [ + ... + ], + "edges": [ + ... + ] + }, + "tooltip": {}, + "relationIdMap": { + ... + }, + "listIdMap": { + ... + } + } + }, + "sessionId": "6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + } + ``` +* **/sqlflow/generation/sqlflow/getSelectedDbObjectInfo** + * Description: get the selected dbobject information, such as file information, sql index, dbobject positions, sql which contains selected dbobject. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **sessionId**: request sessionId, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * **coordinates**: the select dbobject positions, it's a json array string, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * session id: `6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051` + * coordinates: `[{'x':1,'y':8,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'},{'x':1,'y':12,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'}]` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/getSelectedDbObjectInfo" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "coordinates=[{'x':1,'y':8,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'},{'x':1,'y':12,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'}]" -F "sessionId=6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + ``` + * response: + ```json + { + "code": 200, + "data": [ + { + "index": 0, + "positions": [ + { + "x": 1, + "y": 8 + }, + { + "x": 1, + "y": 12 + } + ], + "sql": "select name from user" + } + ] + } + ``` + +### Sqlflow User Job Interface +* **/sqlflow/job/submitUserJob** + * Description: submit user job for multiple sql files, support zip file. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobName**: job name, required **true** + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * **sqlfiles**: request sql files, please use **multiple parts** to submit the sql files, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * curl command: **Note: please add **@** before the sql file path** + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/submitUserJob" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "sqlfiles=@D:/sql.txt" -F "dbvendor=dbvoracle" -F "jobName=job_test" + ``` + * response: + ```json + { + "code": 200, + "data": { + "jobId": "6218721f092540c5a771ca8f82986be7", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "defaultDatabase": "", + "defaultSchema": "", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:11:28", + "status": "create" + } + } + ``` + +* **/sqlflow/job/displayUserJobsSummary** + * Description: get the user jobs summary information. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/displayUserJobsSummary" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" + ``` + * response: + ```json + { + "code": 200, + "data": { + "total": 1, + "success": 1, + "partialSuccess": 0, + "fail": 0, + "jobIds": [ + "bb996c1ee5b741c5b4ff6c2c66c371dd" + ], + "jobDetails": [ + { + "jobId": "bb996c1ee5b741c5b4ff6c2c66c371dd", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:16:11", + "status": "success", + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + ] + } + } + ``` + +* **/sqlflow/job/displayUserJobSummary** + * Description: get the specify user job information. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user jobs summary detail, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/displayUserJobSummary" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" + ``` + * response: + ```json + { + "code": 200, + "data": { + "total": 1, + "success": 1, + "partialSuccess": 0, + "fail": 0, + "jobIds": [ + "bb996c1ee5b741c5b4ff6c2c66c371dd" + ], + "jobDetails": [ + { + "jobId": "bb996c1ee5b741c5b4ff6c2c66c371dd", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:16:11", + "status": "success", + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + ] + } + } + ``` + +* **/sqlflow/job/deleteUserJob** + * Description: delete the user job by job id. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user job detail, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/deleteUserJob" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" + ``` + * response: + ```json + { + "code": 200, + "data": { + "jobId": "bb996c1ee5b741c5b4ff6c2c66c371dd", + "jobName": "job_test", + "userId": "user_test", + "dbVendor": "dbvoracle", + "fileNames": [ + "sql.txt" + ], + "createTime": "2020-09-08 10:16:11", + "status": "delete", + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + } + ``` + +* **/sqlflow/job/displayUserJobGraph** + * Description: get the sqlflow job's model and graph + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user jobs summary detail, required **true** + * database: selected database, required false + * schema: selected schema, required false + * table: selected table, required false + * isReturnModel: whether return the sqlflow model, required false, default value is true + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether output relation simply, required false, default value is false + * ignoreRecordSet: whether ignore the record sets, required false, default value is false + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/displayUserJobGraph?showRelationType=fdd&showRelationType=" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" -F "ignoreFunction=true" -F "ignoreRecordSet=true" -F "isReturnModel=false" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" -F "table=user" + ``` + * response: + ```json + { + "code": 200, + "data": { + "mode": "global", + "summary": { + ... + }, + "graph": { + "elements": { + "tables": [ + ... + ], + "edges": [ + ... + ], + }, + "tooltip": {}, + "relationIdMap": { + ... + }, + "listIdMap": { + ... + } + } + }, + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + ``` + +* **/sqlflow/job/updateUserJobGraphCache** + * Description: update the user job graph cache, then user can call **/sqlflow/generation/sqlflow/selectedgraph** by sessionId, the sessionId value is from job detail. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **jobId**: job id, the value is from user job detail, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql file: D:\sql.txt + * job id: bb996c1ee5b741c5b4ff6c2c66c371dd + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/job/updateUserJobGraphCache" -H "Request-Origion:SwaggerBootstrapUi" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "jobId=bb996c1ee5b741c5b4ff6c2c66c371dd" + ``` + * response: + ```json + { + "code": 200, + "data": { + "sessionId": "a9f751281f8ef6936c554432e169359190d392565208931f201523e08036109d_1599531372233" + } + } + ``` +## Swagger +More information, please check the test environment swagger document: + + * http://111.229.12.71:8081/gspLive_backend/doc.html?lang=en + * Token: `eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYwMzc1NjgwMCwiaWF0IjoxNTcyMjIwODAwfQ.EhlnJO7oqAHdr0_bunhtrN-TgaGbARKvTh2URTxu9iU` diff --git a/api/sqlflow_api_tutorial.md b/api/sqlflow_api_tutorial.md new file mode 100644 index 0000000..329719a --- /dev/null +++ b/api/sqlflow_api_tutorial.md @@ -0,0 +1,51 @@ +## How to use Rest API of SQLFlow + +This article describes how to use the Rest API provided by the SQLFlow to +communicate with the SQLFlow server and get the generated metadata and data lineage. + +In this article, we use `Curl` to demonstrate the usage of the Rest API, +you can use any preferred programming language as you like. + +### Prerequisites +To use the Rest API of the SQLFlow, you need to obtain a premium account. +After that, you will get the `userid` and `secret key`, which will be used in the API. + +- User ID +- Secrete Key + +### Call Rest API + +#### 1. Generate a token + +Once you have the `userid` and `secret key`, the first API need to call is: + +``` +/gspLive_backend/user/generateToken +``` + +This API will return a temporary token that needs to be used in the API call thereafter. + +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/user/generateToken" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:application/x-www-form-urlencoded;charset=UTF-8" -d "secretKey=YOUR SECRET KEY" -d "userId=YOUR USER ID HERE" +``` + + +#### 2. Generate the data lineage + +Call this API by sending the SQL query and get the result includes the data lineage. + +``` +/gspLive_backend/sqlflow/generation/sqlflow +``` + +Example in `Curl` +``` +curl -X POST "https://api.gudusoft.com/gspLive_backend/sqlflow/generation/sqlflow?showRelationType=fdd" -H "Request-Origion:testClientDemo" -H "accept:application/json;charset=utf-8" -H "Content-Type:multipart/form-data" -F "sqlfile=" -F "dbvendor=dbvoracle" -F "ignoreRecordSet=false" -F "simpleOutput=false" -F "sqltext=CREATE VIEW vsal as select * from emp" -F "userId=YOUR USER ID HERE" -F "token=YOUR TOKEN HERE" +``` + +#### 3. Other features +You can also use the rest API to submit a zip file that includes many SQL files or generate a map of the columns in the join condition. + +### The full reference to the Rest APIs + +[SQLFlow rest API reference](sqlflow_api.md) diff --git a/api/sqlflow_web_ui_control.md b/api/sqlflow_web_ui_control.md new file mode 100644 index 0000000..c98bded --- /dev/null +++ b/api/sqlflow_web_ui_control.md @@ -0,0 +1,263 @@ +# SQLFlow Web UI Control + +![SQLFlow Control](../images/sqlflow_web_ui_control.png) + +SQLFlow Web UI has some choice to control the result: + +1. hide all columns + * just affect ui, table column ui height is 0. +2. dataflow + * show fdd relation. +3. impact + * show fdr, fdr relations. +4. show intermediate recordset + * display or hide intermediate recordset +5. show function + * display or hide function + +## Web API Call +We use the restful api **/sqlflow/generation/sqlflow/graph** to get the sqlflow graph, it has several arguments: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * sqltext: sql text, required false + * sqlfile: sql file, required false + * **dbvendor**: database vendor, required **true**, available values: + * dbvbigquery, dbvcouchbase,dbvdb2,dbvgreenplum,dbvhana,dbvhive,dbvimpala,dbvinformix,dbvmdx,dbvmysql,dbvnetezza,dbvopenedge,dbvoracle,dbvpostgresql,dbvredshift,dbvsnowflake,dbvmssql,dbvsybase,dbvteradata,dbvvertica + * showRelationType: show relation type, required false, default value is **fdd**, multiple values seperated by comma like fdd,frd,fdr. Available values: + * **fdd**: value of target column from source column + * **frd**: the recordset count of target column which is affect by value of source column + * **fdr**: value of target column which is affected by the recordset count of source column + * **join**: combine rows from two or more tables, based on a related column between them + * simpleOutput: whether output relation simply, required false, default value is false + * ignoreRecordSet: whether ignore the record sets, required false, default value is false + * showLinkOnly: whether show relation linked columns only, required false, default value is true + * hideColumn: whether hide the column ui, required false, default value is false + * ignoreFunction: whether ignore the function relations, required false, default value is false + +## How to Control The Sqlflow Web UI +1. hide all columns + * it matches the `hideColumn` argument. If the argument is `true`, `hideColumn` will be checked. +2. dataflow + * it matches the `showRelationType` argument. If the argument contains `fdd`, `dataflow` will be checked. +3. impact + * it matches the `showRelationType` argument. If the argument contains `fdr,fdd`, `impact` will be checked. +4. show intermediate recordset + * it matches the `ignoreRecordSet` argument. If the argument is `true`, `show intermediate recordset` will be checked. +5. show function + * it matches the `ignoreFunction` argument. If the argument is `true`, `show function` will be checked. + +![SQLFlow Join](../images/sqlflow_web_ui_join.png) + +1. Visualize join + * show join relations. + * it matches the `showRelationType` argument. If the argument is `join`, `Visualize join` will be displayed. + +![SQLFlow Error Message](../images/sqlflow_error_message.png) + +If sqlflow has some errors, it will be shown in the sqlflow json. +Sqlflow error message has 4 types: + * SYNTAX_ERROR + * gsp parsing sql returns some error messages. + * SYNTAX_HINT + * gsp parsing sql returns some hint messages. + * ANALYZE_ERROR + * dataflow analyzer occurs error. + * LINK_ORPHAN_COLUMN + * dataflow analyzer returns linking orphan column hint. + +## Get the Error Message Position +Typically, if the datafow returns error messages, the lineage xml will show: +```xml + + ... + + +``` +Noting `coordinate="[4,22,0],[4,30,0]"`, we can use it to get the error position. [4,22,0] is the start position and [4,30,0] is the end position, 0 is the index of SQLInfo hashcode. + +## How to Use WebAPI to Point the Position +* **/sqlflow/generation/sqlflow/getSelectedDbObjectInfo** + * Description: get the selected dbobject information, such as file information, sql index, dbobject positions, sql which contains selected dbobject. + * HTTP Method: **POST** + * Parameters: + * **userId**: the user id of sqlflow web or client, required **true** + * **token**: the token of sqlflow client request. sqlflow web, required false, sqlflow client, required true + * **sessionId**: request sessionId, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * **coordinates**: the select dbobject positions, it's a json array string, the value is from api **/sqlflow/generation/sqlflow/graph**, required **true** + * Return code: + * 200: successful + * other: failed, check the error field to get error message. + * Sample: + * test sql: + ```sql + select name from user + ``` + * session id: `6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051` + * coordinates: `[{'x':1,'y':8,'hashCode':'0'},{'x':1,'y':12,'hashCode':'0'}]` + * curl command: + ```bash + curl -X POST "http://127.0.0.1:8081/gspLive_backend/sqlflow/generation/sqlflow/getSelectedDbObjectInfo" -H "accept:application/json;charset=utf-8" -F "userId=google-oauth2|104002923119102769706" -F "token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYxMDEyMTYwMCwiaWF0IjoxNTc4NTg1NjAwfQ.9AAIkjZ3NF7Pns-hRjZQqRHprcsj1dPKHquo8zEp7jE" -F "coordinates=[{'x':1,'y':8,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'},{'x':1,'y':12,'hashCode':'3630d5472af5f149fe3fb2202c8a338d'}]" -F "sessionId=6172a4095280ccce97e996242d8b4084f46e2c954455e71339aeffccad5f0d57_1599501562051" + ``` + * response: + ```json + { + "code": 200, + "data": [ + { + "index": 0, + "positions": [ + { + "x": 1, + "y": 8 + }, + { + "x": 1, + "y": 12 + } + ], + "sql": "select name from user" + } + ] + } + ``` + +## Get SQL Information By SQLFLow Coordinate + +### SQLInfo +When the sqlflow analyzing sql has been finished, it recorded some sql information, we can use it to locate database object position. + +```java +public class SqlInfo { + private String fileName; + private String sql; + private int originIndex; + private int index; + private String group; + private int originLineStart; + private int originLineEnd; + private int lineStart; + private int lineEnd; + private String hash; +} +``` + +Each sql file matches a SqlInfo object, and the map key is "hash" property. + +Sqlflow provides a tool class **gudusoft.gsqlparser.dlineage.util.SqlInfoHelper**, which can transform dataflow coordinate to `DbObjectPosition`. + +### SqlInfoHelper + +1. First step, call api `SqlInfoHelper.getSqlInfoJson` to fetch the sqlinfo map from the DataFlowAnalyzer object, and persist it. +```java + public static String getSqlInfoJson(DataFlowAnalyzer analyzer); +``` + +2. Second step, initialize the SqlInfoHelper with the sqlinfo json string. +```java + //Constructor + public SqlInfoHelper(String sqlInfoJson); +``` + +3. Third step, transform sqlflow position string to `dataflow.model.json.Coordinate` array. + * If you use the `dataflow.model.json.DataFlow` model, you can get the Coordinate object directly, doesn't need any transform. + * If you use the `dataflow.model.xml.dataflow` model, you can call api `SqlInfoHelper.parseCoordinateString` + ```java + public static Coordinate[][] parseCoordinateString(String coordinate); + ``` + * Method parseCoordinateString support both of xml output coordinate string and json output coordinate string, like these: + ``` + //xml output coordinate string + [56,36,0],[56,62,0] + + //json output coordinate string + [{"x":31,"y":36,"hashCode":"0"},{"x":31,"y":38,"hashCode":"0"}] + ``` + +4. Fourth step, get the DbObjectPosition by api `getSelectedDbObjectInfo` +```java + public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end); +``` + * Each position has two coordinates, start coordinate and end coordinate. If the result of DBObject.getCoordinates() has 10 items, it matches 5 positions. + * The position is based on the entire file, but not one statement. + * The sql field of DbObjectPosition return all sqls of the file. + +5. If you just want to get the specific statement information, please call the api `getSelectedDbObjectStatementInfo` +```java + public DbObjectPosition getSelectedDbObjectStatementInfo(EDbVendor vendor, Coordinate start, Coordinate end); +``` + * The position is based on the statement. + * Return the statement index of sqls, index **bases 0**. + * Return a statement, but not all sqls of the file. + +### How to Use DbObjectPosition +```java +public class DbObjectPosition { + private String file; + private String sql; + private int index; + private List> positions = new ArrayList>(); +} +``` +* file field matches the sql file name. +* sql field matches the sql content. +* index: + * If the sql file is from `grabit`, it's a json file, and it has an json array named "query", the value of index field is the query item index. + * Other case, the value of index field is 0. +* positions, locations of database object, they are matched the sql field. Position x and y **base 1** but not 0. + +### Example 1 (getSelectedDbObjectInfo) +```java + String sql = "Select\n a\nfrom\n b;"; + DataFlowAnalyzer dataflow = new DataFlowAnalyzer(sql, EDbVendor.dbvmssql, false); + dataflow.generateDataFlow(new StringBuffer()); + dataflow flow = dataflow.getDataFlow(); + String coordinate = flow.getTables().get(0).getCoordinate(); + Coordinate[][] coordinates = SqlInfoHelper.parseCoordinateString(coordinate); + SqlInfoHelper helper = new SqlInfoHelper(SqlInfoHelper.getSqlInfoJson(dataflow)); + DbObjectPosition position = helper.getSelectedDbObjectInfo(coordinates[0][0], coordinates[0][1]); + System.out.println(position.getSql()); + System.out.println("table " + flow.getTables().get(0).getName() + " position is " + Arrays.toString(position.getPositions().toArray())); +``` + +Return: +```java +Select + a +from + b; + +table b position is [[4,2], [4,3]] +``` + +### Example 2 (getSelectedDbObjectStatementInfo) +```java + String sql = "Select\n a\nfrom\n b;\n Select c from d;"; + DataFlowAnalyzer dataflow = new DataFlowAnalyzer(sql, EDbVendor.dbvmssql, false); + dataflow.generateDataFlow(new StringBuffer()); + gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow flow = dataflow.getDataFlow(); + String coordinate = flow.getTables().get(1).getCoordinate(); + Coordinate[][] coordinates = SqlInfoHelper.parseCoordinateString(coordinate); + SqlInfoHelper helper = new SqlInfoHelper(SqlInfoHelper.getSqlInfoJson(dataflow)); + DbObjectPosition position = helper.getSelectedDbObjectStatementInfo(EDbVendor.dbvmssql, coordinates[0][0], coordinates[0][1]); + System.out.println(position.getSql()); + System.out.println( + "table " + flow.getTables().get(1).getName() + " position is " + Arrays.toString(position.getPositions().toArray())); + System.out.println( + "stmt index is " + position.getIndex()); +``` + +Return: +```java +Select c from d; +table d position is [[1,20], [1,21]] +stmt index is 1 +``` + + + + + + + + + \ No newline at end of file diff --git a/assets/images/discover-data-lineage-from-subquery-and-cte1.png b/assets/images/discover-data-lineage-from-subquery-and-cte1.png new file mode 100644 index 0000000..8219f7c Binary files /dev/null and b/assets/images/discover-data-lineage-from-subquery-and-cte1.png differ diff --git a/assets/images/get-started-11-variables1.png b/assets/images/get-started-11-variables1.png new file mode 100644 index 0000000..81132ba Binary files /dev/null and b/assets/images/get-started-11-variables1.png differ diff --git a/assets/images/get-started-intermediate-resultset1.png b/assets/images/get-started-intermediate-resultset1.png new file mode 100644 index 0000000..0057acd Binary files /dev/null and b/assets/images/get-started-intermediate-resultset1.png differ diff --git a/assets/images/get-started-intermediate-resultset2.png b/assets/images/get-started-intermediate-resultset2.png new file mode 100644 index 0000000..7d7f883 Binary files /dev/null and b/assets/images/get-started-intermediate-resultset2.png differ diff --git a/assets/images/get-started-intermediate-resultset3.png b/assets/images/get-started-intermediate-resultset3.png new file mode 100644 index 0000000..1ea5c06 Binary files /dev/null and b/assets/images/get-started-intermediate-resultset3.png differ diff --git a/assets/images/get-started-transform1.png b/assets/images/get-started-transform1.png new file mode 100644 index 0000000..198a5f9 Binary files /dev/null and b/assets/images/get-started-transform1.png differ diff --git a/data-lineage-xml-elements.md b/data-lineage-xml-elements.md new file mode 100644 index 0000000..a467b96 --- /dev/null +++ b/data-lineage-xml-elements.md @@ -0,0 +1,58 @@ +## Element in the data lineage xml output generated by the SQLFlow + +### Table +`Table` is one of the major elements in the output of the data lineage. + +The `type` of a `table` element can be the value of `table`, `pseudoTable` + +#### 1. type = "table" +This means a base table found in the SQL query. + +```sql +create view v123 as select a,b from employee a, name b where employee.id = name.id +``` + +```xml + +``` + +#### 2. type = "pseudoTable" +Due to the lack of metadata information, some columns can't be linked to a table correctly. +Those columns will be assigned to a pseudo table with name: `pseudo_table_include_orphan_column`. +The type of this table is `pseudoTable`. + +In the following sample sql, columm `a`, `b` can't be linked to a specific table without enough information, +so a pseudo table with name `pseudo_table_include_orphan_column` is created to contain those orphan columns. + +```sql +create view v123 as select a,b from employee a, name b where employee.id = name.id +``` + +```xml +
+ + +
+``` + +#### tableType +In the most case of SQL query, the table used is a base table. +However, derived tables are also used in the from clause or other places. + +The `tableType` property in the `table` element tells you what kind of the derived table this table is. + +Take the following sql for example, `WarehouseReporting.dbo.fnListToTable` is a function that +used as a derived table. So, the value of `tableType` is `function`. + +Currently(GSP 2.2.0.6), `function` is the only value of `tableType`. More value of `tableType` will be added in the later version +such as `JSON_TABLE` for JSON_TABLE. + +```sql +select entry as Account FROM WarehouseReporting.dbo.fnListToTable(@AccountList) +``` + +```xml + + +
+``` diff --git a/databases/azuresql/grabit-azure-1.png b/databases/azuresql/grabit-azure-1.png new file mode 100644 index 0000000..89c4ef6 Binary files /dev/null and b/databases/azuresql/grabit-azure-1.png differ diff --git a/databases/azuresql/grabit-azure-2-database.png b/databases/azuresql/grabit-azure-2-database.png new file mode 100644 index 0000000..435cf5e Binary files /dev/null and b/databases/azuresql/grabit-azure-2-database.png differ diff --git a/databases/azuresql/grabit-azure-3-database-parameters.png b/databases/azuresql/grabit-azure-3-database-parameters.png new file mode 100644 index 0000000..d8af62c Binary files /dev/null and b/databases/azuresql/grabit-azure-3-database-parameters.png differ diff --git a/databases/azuresql/grabit-azure-4-sqlflow.png b/databases/azuresql/grabit-azure-4-sqlflow.png new file mode 100644 index 0000000..f8cc3ed Binary files /dev/null and b/databases/azuresql/grabit-azure-4-sqlflow.png differ diff --git a/databases/azuresql/grabit-azure-5-sqlflow-result.png b/databases/azuresql/grabit-azure-5-sqlflow-result.png new file mode 100644 index 0000000..c310bdf Binary files /dev/null and b/databases/azuresql/grabit-azure-5-sqlflow-result.png differ diff --git a/databases/azuresql/grabit-azure-6-data-lineage-result.png b/databases/azuresql/grabit-azure-6-data-lineage-result.png new file mode 100644 index 0000000..99cb939 Binary files /dev/null and b/databases/azuresql/grabit-azure-6-data-lineage-result.png differ diff --git a/databases/azuresql/grabit-azure-command-line.md b/databases/azuresql/grabit-azure-command-line.md new file mode 100644 index 0000000..e097bd4 --- /dev/null +++ b/databases/azuresql/grabit-azure-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from Azure (Command Line Mode) +This article introduces how to discover the data lineage from azure scripts or the azure database and automatically update it. +So the business users and developers can see the azure data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a Azure database +- Modify the `conf-template\azure-config-template` to meet your environment. + +Here is a sample config file: `azure-config` that grabs metadata from the remote azure database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseType":"azure", + "optionType":1, + "resultType":1, + "databaseServer":{ + "hostname":"azure ip address", + "port":"1433", + "username":"azure user name", + "password":"your password here", + "database":"", + "extractedDbsSchemas":"", + "excludedDbsSchemas":"", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f azure-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/azuresql/readme.md b/databases/azuresql/readme.md new file mode 100644 index 0000000..f9fee50 --- /dev/null +++ b/databases/azuresql/readme.md @@ -0,0 +1,68 @@ +## Automated data lineage from Azure (GUI Mode) +This article introduces how to discover the data lineage from azure scripts or the azure database and automatically update it. +So the business users and developers can see the Azure data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a Azure database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit azure UI 1](grabit-azure-1.png) + +- Select `azure` in the list + +![Grabit azure UI 2 database](grabit-azure-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in DEMO_DB/PUBLIC schema. + +![Grabit snowfalke UI 3 database parameters](grabit-azure-3-database-parameters.png) + +- note + +1.The `Database` parameter is must specified. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for the currently connected database is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + + +- After grabbing the metadata from the azure database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit azure SQLFlow](grabit-azure-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit azure SQLFlow result](grabit-azure-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit azure data lineage result](grabit-azure-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a Azure database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover azure data lineage in command line mode](grabit-azure-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a azure database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) diff --git a/databases/connect-to-databases.md b/databases/connect-to-databases.md new file mode 100644 index 0000000..4397e32 --- /dev/null +++ b/databases/connect-to-databases.md @@ -0,0 +1,244 @@ +## Grabit Databse Connection Information Document + +Specify a database instance that grabit will connect to fetch the metadata that helps SQLFlow make a more precise analysis and get a more accurate result of data lineage. + +#### Databse Connection Information UI +![Databse Connection Information UI](connection.jpg) + +#### Parameter Specification Of Connection Information + +#### hostname + +The IP of the database server that the grabit connects. + +#### port + +The port number of the database server that the grabit connect. + +#### username + +The database user used to login to the database. + +#### password + +The password of the database user. + +note: the passwords can be encrypted using tools [Encrypted password](#Encrypted password), using encrypted passwords more secure. + +#### privateKeyFile + +Use a private key to connect, Only supports the `snowflake`. + +#### privateKeyFilePwd + +Generate the password for the private key, Only supports the `snowflake`. + +#### database + +The name of the database instance to which it is connected. + +For azure,greenplum,netezza,oracle,postgresql,redshift,teradata databases, it represents the database name and is required, For other databases, it is optional. + +` +note: +If this parameter is specified and the database to which it is connected is Azure, Greenplum, PostgreSQL, or Redshift, then only metadata under that library is extracted. +` + +#### extractedDbsSchemas + +List of databases and schemas to extract, separated by +commas, which are to be provided in the format database/schema; +Or blank to extract all databases. +`database1/schema1,database2/schema2,database3` or `database1.schema1,database2.schema2,database3` +When parameter `database` is filled in, this parameter is considered a schema. +And support wildcard characters such as `database1/*`,`*/schema`,`*/*`. + +When the connected databases are `Oracle` and `Teradata`, this parameter is set the schemas, for example: + +````json +extractedDbsSchemas: "HR,SH" +```` + +When the connected databases are `Mysql` , `Sqlserver`, `Postgresql`, `Snowflake`, `Greenplum`, `Redshift`, `Netezza`, `Azure`, this parameter is set database/schema, for example: + +````json +extractedDbsSchemas: "MY/ADMIN" +```` + + +#### excludedDbsSchemas + +This parameters works under the resultset filtered by `extractedDbsSchemas`. +List of databases and schemas to exclude from extraction, separated by commas +`database1/schema1,database2` or `database1.schema1,database2` +When parameter `database` is filled in, this parameter is considered a schema. +And support wildcard characters such as `database1/*`,`*/schema`,`*/*`. + +When the connected databases are `Oracle` and `Teradata`, this parameter is set the schemas, for example: + +````json +excludedDbsSchemas: "HR" +```` + +When the connected databases are `Mysql` , `Sqlserver`, `Postgresql`, `Snowflake`, `Greenplum`, `Redshift`, `Netezza`, `Azure`, this parameter is set database/schema, for example: + +````json +excludedDbsSchemas: "MY/*" +```` + +#### extractedStoredProcedures + +A list of stored procedures under the specified database and schema to extract, separated by +commas, which are to be provided in the format database.schema.procedureName or schema.procedureName; +Or blank to extract all databases, support expression. +`database1.schema1.procedureName1,database2.schema2.procedureName2,database3.schema3,database4` or `database1/schema1/procedureName1,database2/schema2` + +for example: + +````json +extractedStoredProcedures: "database.scott.vEmp*" +```` + +or + +````json +extractedStoredProcedures: "database.scott" +```` + +#### extractedViews + +A list of stored views under the specified database and schema to extract, separated by +commas, which are to be provided in the format database.schema.viewName or schema.viewName. +Or blank to extract all databases, support expression. +`database1.schema1.procedureName1,database2.schema2.procedureName2,database3.schema3,database4` or `database1/schema1/procedureName1,database2/schema2` + +for example: + +````json +extractedViews: "database.scott.vEmp*" +```` + +or + +````json +extractedViews: "database.scott" +```` + +#### enableQueryHistory + +Fetch SQL queries from the query history if set to `true` default is false. + +#### queryHistoryBlockOfTimeInMinutes + +When `enableQueryHistory:true`, the interval at which the SQL query was extracted in the query History,default is `30` minutes. + +#### queryHistorySqlType + +When `enableQueryHistory:true`, the DML type of SQL is extracted from the query History. +When empty, all types are extracted, and when multiple types are specified, a comma separates them, such as `SELECT,UPDATE,MERGE`. +Currently only the snowflake database supports this parameter,support types are **SHOW,SELECT,INSERT,UPDATE,DELETE,MERGE,CREATE TABLE, CREATE VIEW, CREATE PROCEDURE, CREATE FUNCTION**. + +for example: + +````json +queryHistorySqlType: "SELECT,DELETE" +```` + +#### snowflakeDefaultRole + +This value represents the role of the snowflake database. + +```` +note: You must define a role that has access to the SNOWFLAKE database,And assign WAREHOUSE permission to this role. +```` + +Assign permissions to a role, for example: + +````sql +#create role +use role accountadmin; +grant imported privileges on database snowflake to role sysadmin; +grant imported privileges on database snowflake to role customrole1; +use role customrole1; +select * from snowflake.account_usage.databases; + +#To do this, the Role gives the WAREHOUSE permission +select current_warehouse() +use role sysadmin +GRANT ALL PRIVILEGES ON WAREHOUSE %current_warehouse% TO ROLE customrole1; +```` + +#### metaStore + +If the current data source is a `Hive` or `Spark` data store, this parameter can be set to `hive` or `sparksql`. By default, this parameter is left blank. + + + +Sample configuration of a SQL Server database: +```json +"hostname":"127.0.0.1", +"port":"1433", +"username":"sa", +"password":"PASSWORD", +"database":"", +"extractedDbsSchemas":"AdventureWorksDW2019/dbo", +"excludedDbsSchemas":"", +"extractedStoredProcedures":"AdventureWorksDW2019.dbo.f_qry*", +"extractedViews":"", +"enableQueryHistory":false, +"queryHistoryBlockOfTimeInMinutes":30, +"snowflakeDefaultRole":"", +"queryHistorySqlType":"", +"metaStore":"hive" +``` + +#### sqlsourceTableName + +table name: **query_table** + +| query_name | query_source | +| ---------- | ----------------------------------- | +| query1 | create view v1 as select f1 from t1 | +| query2 | create view v2 as select f2 from t2 | +| query3 | create view v3 as select f3 from t3 | + +If you save SQL queries in a specific table, one SQL query per row. + +Let's say: `query_table.query_source` store the source code of the query. +We can use this query to fetch all SQL queries in this table: + +```sql +select query_name as queryName, query_source as querySource from query_table +``` + +By setting the value of `sqlsourceTableName` and `sqlsourceColumnQuerySource`,`sqlsourceColumnQueryName` +grabit can fetch all SQL queries in this table and send it to the SQLFlow to analzye the lineage. + +In this example, +``` +"sqlsourceTableName":"query_table" +"sqlsourceColumnQuerySource":"query_source" +"sqlsourceColumnQueryName":"query_name" +``` + +Please leave `sqlsource_table_name` empty if you don't fetch SQL queries from a specific table. + +#### sqlsourceColumnQuerySource +In the above sample: +``` +"sqlsourceColumnQuerySource":"query_source" +``` + +#### sqlsourceColumnQueryName +``` +"sqlsourceColumnQueryName":"query_name" +``` +This parameter is optional, you don't need to speicify a query name column if it doesn't exist in the table. + +- **fetch from query history** + +Fetch SQL queries from the query history if set to `yes` default is no, SQL statement that can retrieve history execution from the database to which it is connected. You can specify the time for history execution. The default is 30 minutes. + +` +note: Currently only supported Snowflake,Sqlserver +` diff --git a/databases/connection.jpg b/databases/connection.jpg new file mode 100644 index 0000000..2bfee00 Binary files /dev/null and b/databases/connection.jpg differ diff --git a/databases/greenplum/readme.md b/databases/greenplum/readme.md new file mode 100644 index 0000000..bc2d8ca --- /dev/null +++ b/databases/greenplum/readme.md @@ -0,0 +1 @@ +## Greenplum \ No newline at end of file diff --git a/databases/hive/alter_table_set_location.md b/databases/hive/alter_table_set_location.md new file mode 100644 index 0000000..cec711a --- /dev/null +++ b/databases/hive/alter_table_set_location.md @@ -0,0 +1,29 @@ +### Discover data lineage from Hive alter table set location + +```sql +ALTER TABLE a.b SET LOCATION 's3://xxx/xx/1/xxx/'; +``` +#### output lineage in diagram +![Hive alter table set location data lineage](alter_table_set_location_data_lineage.png) + +#### output lineage in xml +```xml + + + + + + + + +
+ + + + + +``` + +This data lineage in xml is generated by [Gudu SQLFlow Java tool](https://www.gudusoft.com/sqlflow-java-library-2/) + + diff --git a/databases/hive/alter_table_set_location_data_lineage.png b/databases/hive/alter_table_set_location_data_lineage.png new file mode 100644 index 0000000..583108e Binary files /dev/null and b/databases/hive/alter_table_set_location_data_lineage.png differ diff --git a/databases/hive/readme.md b/databases/hive/readme.md new file mode 100644 index 0000000..7e79298 --- /dev/null +++ b/databases/hive/readme.md @@ -0,0 +1,62 @@ +## Hive data lineage examples + +- [Alter table set location](alter_table_set_location.md) + + +## connect to hive metastore + +Use grabit command line to connect to a MySQL database that save the +Hive metastore. Fetch the metadata from the Hive metastore and send +to the SQLFlow to analyze the data lineage. + +### config file +```json +{ + "databaseServer":{ + "hostname":"", + "port":"3306", + "username":"", + "password":"", + "database":"", + "extractedDbsSchemas":"", + "excludedDbsSchemas":"", + "extractedStoredProcedures":"", + "extractedViews":"", + "metaStore":"hive" + }, + "SQLFlowServer":{ + "server":"http://127.0.0.1", + "serverPort":"8081", + "userId":"gudu|0123456789", + "userSecret":"" + }, + "SQLScriptSource":"database", + "lineageReturnFormat":"json", + "databaseType":"mysql" +} +``` + +Please make sure to setup the `database` to the name of the MySQL database +which store the Hive metastore. + +The IP below should be the machine where the SQLFlow on-premise version is installed. +``` +"server":"http://127.0.0.1", +``` + + +### command line syntax +- **mac & linux** +```shell script +chmod +x start.sh + +sh start.sh /f config.json +``` + +- **windows** +```bat +start.bat /f config.json +``` + +## download the latest version grabit tool +https://www.gudusoft.com/grabit/ diff --git a/databases/mysql/grabit-mysql-1.png b/databases/mysql/grabit-mysql-1.png new file mode 100644 index 0000000..3ed38c6 Binary files /dev/null and b/databases/mysql/grabit-mysql-1.png differ diff --git a/databases/mysql/grabit-mysql-2-database.png b/databases/mysql/grabit-mysql-2-database.png new file mode 100644 index 0000000..03b4241 Binary files /dev/null and b/databases/mysql/grabit-mysql-2-database.png differ diff --git a/databases/mysql/grabit-mysql-3-database-parameters.png b/databases/mysql/grabit-mysql-3-database-parameters.png new file mode 100644 index 0000000..883c135 Binary files /dev/null and b/databases/mysql/grabit-mysql-3-database-parameters.png differ diff --git a/databases/mysql/grabit-mysql-4-sqlflow.png b/databases/mysql/grabit-mysql-4-sqlflow.png new file mode 100644 index 0000000..815da51 Binary files /dev/null and b/databases/mysql/grabit-mysql-4-sqlflow.png differ diff --git a/databases/mysql/grabit-mysql-5-sqlflow-result.png b/databases/mysql/grabit-mysql-5-sqlflow-result.png new file mode 100644 index 0000000..85671a5 Binary files /dev/null and b/databases/mysql/grabit-mysql-5-sqlflow-result.png differ diff --git a/databases/mysql/grabit-mysql-6-data-lineage-result.png b/databases/mysql/grabit-mysql-6-data-lineage-result.png new file mode 100644 index 0000000..fa79b89 Binary files /dev/null and b/databases/mysql/grabit-mysql-6-data-lineage-result.png differ diff --git a/databases/mysql/grabit-mysql-command-line.md b/databases/mysql/grabit-mysql-command-line.md new file mode 100644 index 0000000..41d8334 --- /dev/null +++ b/databases/mysql/grabit-mysql-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from MySQL (Command Line Mode) +This article introduces how to discover the data lineage from MySQL scripts or the MySQL database and automatically update it. +So the business users and developers can see the MySQL data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a MySQL database +- Modify the `conf-template\mysql-config-template` to meet your environment. + +Here is a sample config file: `mysql-config` that grabs metadata from a local MySQL database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseServer":{ + "hostname":"mysql ip address", + "port":"3306", + "username":"mysql user name", + "password":"your password here", + "database":"", + "extractedDbsSchemas":"", + "excludedDbsSchemas":"sys,mysql,performance_schema,information_schema", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "optionType":1, + "resultType":1, + "databaseType":"mysql", + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f mysql-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/mysql/readme.md b/databases/mysql/readme.md new file mode 100644 index 0000000..7c826e5 --- /dev/null +++ b/databases/mysql/readme.md @@ -0,0 +1,72 @@ +## Automated data lineage from MySQL (GUI Mode) +This article introduces how to discover the data lineage from MySQL scripts or the MySQL database and automatically update it. +So the business users and developers can see the MySQL data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a MySQL database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit MySQL UI 1](grabit-mysql-1.png) + +- Select `mysql` in the list + +![Grabit MySQL UI 2 database](grabit-mysql-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in employees database. + +![Grabit MySQL UI 3 database parameters](grabit-mysql-3-database-parameters.png) + +- note + +1.The `Database` parameter is optional. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for all databases is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + +- After grabbing the metadata from the MySQL database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit MySQL SQLFlow](grabit-mysql-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit MySQL SQLFlow result](grabit-mysql-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit MySQL data lineage result](grabit-mysql-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a MySQL database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover MySQL data lineage in command line mode](grabit-mysql-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a MySQL database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) + + + +### Know-How +![sqlflow-automated-data-lineage](/images/sqlflow-overview-grabit.png "SQLFlow automated data lineage") diff --git a/databases/netezza/grabit-netezza-1.png b/databases/netezza/grabit-netezza-1.png new file mode 100644 index 0000000..89c4ef6 Binary files /dev/null and b/databases/netezza/grabit-netezza-1.png differ diff --git a/databases/netezza/grabit-netezza-2-database.png b/databases/netezza/grabit-netezza-2-database.png new file mode 100644 index 0000000..55a2ddf Binary files /dev/null and b/databases/netezza/grabit-netezza-2-database.png differ diff --git a/databases/netezza/grabit-netezza-3-database-parameters.png b/databases/netezza/grabit-netezza-3-database-parameters.png new file mode 100644 index 0000000..785a9f4 Binary files /dev/null and b/databases/netezza/grabit-netezza-3-database-parameters.png differ diff --git a/databases/netezza/grabit-netezza-4-sqlflow.png b/databases/netezza/grabit-netezza-4-sqlflow.png new file mode 100644 index 0000000..547f835 Binary files /dev/null and b/databases/netezza/grabit-netezza-4-sqlflow.png differ diff --git a/databases/netezza/grabit-netezza-5-sqlflow-result.png b/databases/netezza/grabit-netezza-5-sqlflow-result.png new file mode 100644 index 0000000..a749736 Binary files /dev/null and b/databases/netezza/grabit-netezza-5-sqlflow-result.png differ diff --git a/databases/netezza/grabit-netezza-6-data-lineage-result.png b/databases/netezza/grabit-netezza-6-data-lineage-result.png new file mode 100644 index 0000000..431814a Binary files /dev/null and b/databases/netezza/grabit-netezza-6-data-lineage-result.png differ diff --git a/databases/netezza/grabit-netezza-7-data-lineage-result.png b/databases/netezza/grabit-netezza-7-data-lineage-result.png new file mode 100644 index 0000000..1c77696 Binary files /dev/null and b/databases/netezza/grabit-netezza-7-data-lineage-result.png differ diff --git a/databases/netezza/grabit-netezza-command-line.md b/databases/netezza/grabit-netezza-command-line.md new file mode 100644 index 0000000..4d7a42e --- /dev/null +++ b/databases/netezza/grabit-netezza-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from Netezza (Command Line Mode) +This article introduces how to discover the data lineage from netezza scripts or the netezza database and automatically update it. +So the business users and developers can see the netezza data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a Netezza database +- Modify the `conf-template\netezza-config-template` to meet your environment. + +Here is a sample config file: `netezza-config` that grabs metadata from the remote netezza database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseType":"netezza", + "optionType":1, + "resultType":1, + "databaseServer":{ + "hostname":"netezza ip address", + "port":"5480", + "username":"netezza user name", + "password":"your password here", + "database":"MY", + "extractedDbsSchemas":"", + "excludedDbsSchemas":"", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f netezza-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/netezza/readme.md b/databases/netezza/readme.md new file mode 100644 index 0000000..9c40e92 --- /dev/null +++ b/databases/netezza/readme.md @@ -0,0 +1,69 @@ +## Automated data lineage from Netezza (GUI Mode) +This article introduces how to discover the data lineage from netezza scripts or the netezza database and automatically update it. +So the business users and developers can see the SQL Server data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a Netezza database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit netezza UI 1](grabit-netezza-1.png) + +- Select `netezza` in the list + +![Grabit netezza UI 2 database](grabit-netezza-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in DEMO_DB/PUBLIC schema. + +![Grabit snowfalke UI 3 database parameters](grabit-netezza-3-database-parameters.png) + +- note + +1.The `Database` parameter is must specified. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for all databases is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + +- After grabbing the metadata from the netezza database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit netezza SQLFlow](grabit-netezza-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit netezza SQLFlow result](grabit-netezza-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit netezza data lineage result 2](grabit-netezza-7-data-lineage-result.png) + +![Grabit netezza data lineage result 1](grabit-netezza-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a Netezza database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover netezza data lineage in command line mode](grabit-netezza-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a netezza database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) diff --git a/databases/oracle/grabit-oracle-1.png b/databases/oracle/grabit-oracle-1.png new file mode 100644 index 0000000..3ed38c6 Binary files /dev/null and b/databases/oracle/grabit-oracle-1.png differ diff --git a/databases/oracle/grabit-oracle-2-database.png b/databases/oracle/grabit-oracle-2-database.png new file mode 100644 index 0000000..04990ea Binary files /dev/null and b/databases/oracle/grabit-oracle-2-database.png differ diff --git a/databases/oracle/grabit-oracle-3-database-parameters.png b/databases/oracle/grabit-oracle-3-database-parameters.png new file mode 100644 index 0000000..799748a Binary files /dev/null and b/databases/oracle/grabit-oracle-3-database-parameters.png differ diff --git a/databases/oracle/grabit-oracle-4-sqlflow.png b/databases/oracle/grabit-oracle-4-sqlflow.png new file mode 100644 index 0000000..815da51 Binary files /dev/null and b/databases/oracle/grabit-oracle-4-sqlflow.png differ diff --git a/databases/oracle/grabit-oracle-5-sqlflow-result.png b/databases/oracle/grabit-oracle-5-sqlflow-result.png new file mode 100644 index 0000000..8f734fe Binary files /dev/null and b/databases/oracle/grabit-oracle-5-sqlflow-result.png differ diff --git a/databases/oracle/grabit-oracle-6-data-lineage-result.png b/databases/oracle/grabit-oracle-6-data-lineage-result.png new file mode 100644 index 0000000..af2d649 Binary files /dev/null and b/databases/oracle/grabit-oracle-6-data-lineage-result.png differ diff --git a/databases/oracle/grabit-oracle-command-line.md b/databases/oracle/grabit-oracle-command-line.md new file mode 100644 index 0000000..0004a72 --- /dev/null +++ b/databases/oracle/grabit-oracle-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from Oracle (Command Line Mode) +This article introduces how to discover the data lineage from Oracle scripts or the Oracle database and automatically update it. +So the business users and developers can see the Oracle data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a Oracle database +- Modify the `conf-template\oracle-config-template` to meet your environment. + +Here is a sample config file: `oracle-config` that grabs metadata from a local Oracle database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseServer":{ + "hostname":"oracle ip address", + "port":"1521", + "username":"oracle user name", + "password":"your password here", + "database":"oracle database", + "extractSchema":"", + "excludedSchema":"SYS,SYSTEM,OUTLN,MGMT_VIEW,FLOWS_FILES,MDSYS,ORDSYS,EXFSYS,DBSNMP,WMSYS,APPQOSSYS,APEX_030200,OWBSYS_AUDIT,ORDDATA,CTXSYS,ANONYMOUS,SYSMAN,XDB,ORDPLUGINS,OWBSYS,SI_INFORMTN_SCHEMA,OLAPSYS,SCOTT,ORACLE_OCM,MDDATA,DIP,APEX_PUBLIC_USER,SPATIAL_CSW_ADMIN_USR,SPATIAL_WFS_ADMIN_USR", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "optionType":1, + "resultType":1, + "databaseType":"oracle", + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f oracle-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/oracle/readme.md b/databases/oracle/readme.md new file mode 100644 index 0000000..afca072 --- /dev/null +++ b/databases/oracle/readme.md @@ -0,0 +1,72 @@ +## Automated data lineage from Oracle (GUI Mode) +This article introduces how to discover the data lineage from Oracle scripts or the Oracle database and automatically update it. +So the business users and developers can see the Oracle data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a Oracle database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit Oracle UI 1](grabit-oracle-1.png) + +- Select `oracle` in the list + +![Grabit Oracle UI 2 database](grabit-oracle-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in HR schema, and SYS schema the excluded. + +![Grabit Oracle UI 3 database parameters](grabit-oracle-3-database-parameters.png) + +- note + +1.The `Database` parameter is must specified. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for all databases is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + +- After grabbing the metadata from the Oracle database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit Oracle SQLFlow](grabit-oracle-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit Oracle SQLFlow result](grabit-oracle-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit Oracle data lineage result](grabit-oracle-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a Oracle database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover Oracle data lineage in command line mode](grabit-oracle-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a Oracle database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) + + + +### Know-How +![sqlflow-automated-data-lineage](/images/sqlflow-overview-grabit.png "SQLFlow automated data lineage") diff --git a/databases/postgresql/grabit-postgresql-1.png b/databases/postgresql/grabit-postgresql-1.png new file mode 100644 index 0000000..3ed38c6 Binary files /dev/null and b/databases/postgresql/grabit-postgresql-1.png differ diff --git a/databases/postgresql/grabit-postgresql-2-database.png b/databases/postgresql/grabit-postgresql-2-database.png new file mode 100644 index 0000000..4f7bf8f Binary files /dev/null and b/databases/postgresql/grabit-postgresql-2-database.png differ diff --git a/databases/postgresql/grabit-postgresql-3-database-parameters.png b/databases/postgresql/grabit-postgresql-3-database-parameters.png new file mode 100644 index 0000000..5c30faf Binary files /dev/null and b/databases/postgresql/grabit-postgresql-3-database-parameters.png differ diff --git a/databases/postgresql/grabit-postgresql-4-sqlflow.png b/databases/postgresql/grabit-postgresql-4-sqlflow.png new file mode 100644 index 0000000..815da51 Binary files /dev/null and b/databases/postgresql/grabit-postgresql-4-sqlflow.png differ diff --git a/databases/postgresql/grabit-postgresql-5-sqlflow-result.png b/databases/postgresql/grabit-postgresql-5-sqlflow-result.png new file mode 100644 index 0000000..459fba6 Binary files /dev/null and b/databases/postgresql/grabit-postgresql-5-sqlflow-result.png differ diff --git a/databases/postgresql/grabit-postgresql-6-data-lineage-result.png b/databases/postgresql/grabit-postgresql-6-data-lineage-result.png new file mode 100644 index 0000000..701fe33 Binary files /dev/null and b/databases/postgresql/grabit-postgresql-6-data-lineage-result.png differ diff --git a/databases/postgresql/grabit-postgresql-command-line.md b/databases/postgresql/grabit-postgresql-command-line.md new file mode 100644 index 0000000..ba2ee49 --- /dev/null +++ b/databases/postgresql/grabit-postgresql-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from PostgreSQL (Command Line Mode) +This article introduces how to discover the data lineage from PostgreSQL scripts or the PostgreSQL database and automatically update it. +So the business users and developers can see the MySPostgreSQLQL data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a PostgreSQL database +- Modify the `conf-template\postgresql-config-template` to meet your environment. + +Here is a sample config file: `postgresql-config` that grabs metadata from a local PostgreSQL database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseServer":{ + "hostname":"postgresql ip address", + "port":"5432", + "username":"postgresql user name", + "password":"your password here", + "database":"postgresql database", + "extractedDbsSchemas":"", + "excludedDbsSchemas":"information_schema,pg_catalog,public", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "optionType":1, + "resultType":1, + "databaseType":"postgresql", + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f postgresql-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/postgresql/readme.md b/databases/postgresql/readme.md new file mode 100644 index 0000000..a1a4bac --- /dev/null +++ b/databases/postgresql/readme.md @@ -0,0 +1,72 @@ +## Automated data lineage from PostgreSQL (GUI Mode) +This article introduces how to discover the data lineage from PostgreSQL scripts or the PostgreSQL database and automatically update it. +So the business users and developers can see the PostgreSQL data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a PostgreSQL database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit PostgreSQL UI 1](grabit-postgresql-1.png) + +- Select `postgresql` in the list + +![Grabit PostgreSQL UI 2 database](grabit-postgresql-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in public schema. + +![Grabit PostgreSQL UI 3 database parameters](grabit-postgresql-3-database-parameters.png) + +- note + +1.The `Database` parameter is must specified. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for the currently connected database is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + +- After grabbing the metadata from the PostgreSQL database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit PostgreSQL SQLFlow](grabit-postgresql-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit PostgreSQL SQLFlow result](grabit-postgresql-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit PostgreSQL data lineage result](grabit-postgresql-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a PostgreSQL database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover PostgreSQL data lineage in command line mode](grabit-postgresql-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a PostgreSQL database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) + + + +### Know-How +![sqlflow-automated-data-lineage](/images/sqlflow-overview-grabit.png "SQLFlow automated data lineage") diff --git a/databases/readme.md b/databases/readme.md new file mode 100644 index 0000000..d26d5ba --- /dev/null +++ b/databases/readme.md @@ -0,0 +1,29 @@ +## SQLFlow tracks column-level data lineage for more than 20 major databases + +- azure sql server +- bigquery, +- couchbase, +- dax, +- db2, +- **greenplum** +- hana +- **hive** +- impala, +- informix, +- mdx, +- **mysql** +- **netezza** +- odbc +- openedge, +- **oracle** +- **postgresql** +- **redshift** +- **snowflake** +- sparksql, +- **sqlserver** +- sybase +- **teradata** +- vertica + + +### [How to connect to a database](connect-to-databases.md) \ No newline at end of file diff --git a/databases/redshift/readme.md b/databases/redshift/readme.md new file mode 100644 index 0000000..71f5ea4 --- /dev/null +++ b/databases/redshift/readme.md @@ -0,0 +1,44 @@ +# AWS redshift database SQL column-level data lineage + +Discover and visualization lineage from AWS redshift database and SQL script. + +### 1. Copy command + +```sql +copy catdemo +from 's3://awssampledbuswest2/tickit/category_pipe.txt' +iam_role 'arn:aws:iam:::role/' +region 'us-west-2'; +``` + +data lineage diagram: + +[![redshift data lineage copy ](redshift-data-linage-copy.png)](https://sqlflow.gudusoft.com) + + +```sql +copy catdemo(a,b,c) +from 's3://awssampledbuswest2/tickit/category_pipe.txt' +iam_role 'arn:aws:iam:::role/' +region 'us-west-2'; + +``` + +data lineage diagram: + +[![redshift data lineage copy ](redshift-data-linage-copy-with-columns.png)](https://sqlflow.gudusoft.com) + + +### 2. Unload command +```sql +CREATE TABLE a_table(an_int INT, b_int INT); +INSERT INTO a_table VALUES (1,1), (2,1), (3,1), (4,1), (1,2), (2,2), (3,2), (4,2), (5,2), (6,2); + +CREATE TABLE hll_table (sketch HLLSKETCH); +INSERT INTO hll_table select hll_create_sketch(an_int) from a_table group by b_int; + +UNLOAD ('select * from hll_table') TO 's3://mybucket/unload/' +IAM_ROLE 'arn:aws:iam::0123456789012:role/MyRedshiftRole' NULL AS 'null' ALLOWOVERWRITE CSV; +``` + +[![redshift sql data lineage unload](redshift-data-linage-unload.png)](https://sqlflow.gudusoft.com) \ No newline at end of file diff --git a/databases/redshift/redshift-data-linage-copy-with-columns.png b/databases/redshift/redshift-data-linage-copy-with-columns.png new file mode 100644 index 0000000..6501bdc Binary files /dev/null and b/databases/redshift/redshift-data-linage-copy-with-columns.png differ diff --git a/databases/redshift/redshift-data-linage-copy.png b/databases/redshift/redshift-data-linage-copy.png new file mode 100644 index 0000000..fc41159 Binary files /dev/null and b/databases/redshift/redshift-data-linage-copy.png differ diff --git a/databases/redshift/redshift-data-linage-unload.png b/databases/redshift/redshift-data-linage-unload.png new file mode 100644 index 0000000..54c5086 Binary files /dev/null and b/databases/redshift/redshift-data-linage-unload.png differ diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/readme.md b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/readme.md new file mode 100644 index 0000000..ef4eaad --- /dev/null +++ b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/readme.md @@ -0,0 +1,74 @@ +## Snowflake data lineage captured from the script used in bulk Loading from Amazon S3 Using COPY + +### Script for Creating Required Objects +```sql +-- Create a database. A database automatically includes a schema named 'public'. + +create or replace database mydatabase; + +/* Create target tables for CSV and JSON data. The tables are temporary, + meaning they persist only for the duration of the user session and are not visible to other users. +*/ + +create or replace temporary table mycsvtable ( + id integer, + last_name string, + first_name string, + company string, + email string, + workphone string, + cellphone string, + streetaddress string, + city string, + postalcode string); + +create or replace temporary table myjsontable ( + json_data variant); + +-- Create a warehouse + +create or replace warehouse mywarehouse with + warehouse_size='X-SMALL' + auto_suspend = 120 + auto_resume = true + initially_suspended=true; +``` + +### Step 1. Create File Format Objects +```sql +create or replace file format mycsvformat + type = 'CSV' + field_delimiter = '|' + skip_header = 1; +``` + +### Step 2. Create a Named Stage Object +```sql +create or replace stage my_csv_stage + file_format = mycsvformat + url = 's3://snowflake-docs'; +``` + +### Step 3. Copy Data Into the Target Table +```sql +copy into mycsvtable + from @my_csv_stage/tutorials/dataloading/contacts1.csv + on_error = 'skip_file'; +``` + +### Step 4. Verify the Loaded Data +```sql +create materialized view exttable_csv_mv + as + select ID , LAST_NAME , FIRST_NAME ,COMPANY,EMAIL from mycsvtable; +``` + + +## Data lineage build for the above SQL script + +my_csv_stage('s3://snowflake-docs',CSV) -> fdd -> mycsvtable -> fdd -> exttable_csv_mv(ID , LAST_NAME , FIRST_NAME ,COMPANY,EMAIL) +> CSV used in mycsvformat is attached to my_csv_stage + +[![snowflake data lineage bulk loading](snowflake-data-from-stage.png)](https://sqlflow.gudusoft.com) + +[Tutorial: Bulk Loading from Amazon S3 Using COPY](https://docs.snowflake.com/en/user-guide/data-load-external-tutorial.html) diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-copy-into.sql b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-copy-into.sql new file mode 100644 index 0000000..b8c8100 --- /dev/null +++ b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-copy-into.sql @@ -0,0 +1,15 @@ +copy into mycsvtable + from @my_csv_stage/tutorials/dataloading/contacts1.csv + on_error = 'skip_file'; + +/* +copy into mycsvtable + from @my_csv_stage/tutorials/dataloading/ + pattern='.*contacts[1-5].csv' + on_error = 'skip_file'; +*/ + +copy into myjsontable + from @my_json_stage/tutorials/dataloading/contacts.json + on_error = 'skip_file'; + \ No newline at end of file diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-file-format.sql b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-file-format.sql new file mode 100644 index 0000000..8cbd21c --- /dev/null +++ b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-file-format.sql @@ -0,0 +1,9 @@ +create or replace file format mycsvformat + type = 'CSV' + field_delimiter = '|' + skip_header = 1; + + +create or replace file format myjsonformat + type = 'JSON' + strip_outer_array = true; \ No newline at end of file diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-objects.sql b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-objects.sql new file mode 100644 index 0000000..1384603 --- /dev/null +++ b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-objects.sql @@ -0,0 +1,28 @@ +-- Create a database. A database automatically includes a schema named 'public'. + +create or replace database mydatabase; + +/* Create target tables for CSV and JSON data. The tables are temporary, meaning they persist only for the duration of the user session and are not visible to other users. */ + +create or replace temporary table mycsvtable ( + id integer, + last_name string, + first_name string, + company string, + email string, + workphone string, + cellphone string, + streetaddress string, + city string, + postalcode string); + +create or replace temporary table myjsontable ( + json_data variant); + +-- Create a warehouse + +create or replace warehouse mywarehouse with + warehouse_size='X-SMALL' + auto_suspend = 120 + auto_resume = true + initially_suspended=true; \ No newline at end of file diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-stage.sql b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-stage.sql new file mode 100644 index 0000000..12b70d5 --- /dev/null +++ b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/snowflake-create-stage.sql @@ -0,0 +1,12 @@ +create or replace stage my_csv_stage + file_format = mycsvformat + url = 's3://snowflake-docs'; + +create or replace stage my_json_stage + file_format = myjsonformat + url = 's3://snowflake-docs'; + +create or replace stage external_stage + file_format = mycsvformat + url = 's3://private-bucket' + storage_integration = myint; \ No newline at end of file diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/verify_data/snoqflake-create-materialized-view.sql b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/verify_data/snoqflake-create-materialized-view.sql new file mode 100644 index 0000000..af89fc5 --- /dev/null +++ b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/scripts/verify_data/snoqflake-create-materialized-view.sql @@ -0,0 +1,7 @@ +create materialized view exttable_csv_mv + as + select ID , LAST_NAME , FIRST_NAME ,COMPANY,EMAIL from mycsvtable; + +create materialized view exttable_json_mv + as + select * from myjsontable; \ No newline at end of file diff --git a/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/snowflake-data-from-stage.png b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/snowflake-data-from-stage.png new file mode 100644 index 0000000..a4d665e Binary files /dev/null and b/databases/snowflake/bulk-loading-from-amazon-s3-using-copy/snowflake-data-from-stage.png differ diff --git a/databases/snowflake/copy-from-json/readme.md b/databases/snowflake/copy-from-json/readme.md new file mode 100644 index 0000000..079f75b --- /dev/null +++ b/databases/snowflake/copy-from-json/readme.md @@ -0,0 +1,36 @@ +## Snowflake data lineage captured from the script used in bulk Loading from JSON file using COPY + +```sql +create or replace file format json_format + type = 'JSON' + strip_outer_array = true; + +/* Create an internal stage that references the JSON file format. */ + +create or replace stage mystage + file_format = json_format; + +/* Stage the JSON file. */ + +put file:///tmp/sales.json @mystage auto_compress=true; + +/* Create a target table for the JSON data. */ + +create or replace table house_sales (src variant); + +/* Copy the JSON data into the target table. */ + +copy into house_sales + from @mystage/sales.json.gz; + + +create materialized view extjson_mv + as + select * from house_sales; + +``` + +## Data lineage build for the above SQL script +mystage(file:///tmp/sales.json, JSON) -> fdd -> house_sales(src) -> fdd -> extjson_mv + +[![snowflake data lineage copy to json](snowflake-data-linage-copy-to-json.png)](https://sqlflow.gudusoft.com) \ No newline at end of file diff --git a/databases/snowflake/copy-from-json/snowflake-copy-into-single-file-demo.sql b/databases/snowflake/copy-from-json/snowflake-copy-into-single-file-demo.sql new file mode 100644 index 0000000..0e75fbc --- /dev/null +++ b/databases/snowflake/copy-from-json/snowflake-copy-into-single-file-demo.sql @@ -0,0 +1,27 @@ +create or replace file format json_format + type = 'JSON' + strip_outer_array = true; + +/* Create an internal stage that references the JSON file format. */ + +create or replace stage mystage + file_format = json_format; + +/* Stage the JSON file. */ + +put file:///tmp/sales.json @mystage auto_compress=true; + +/* Create a target table for the JSON data. */ + +create or replace table house_sales (src variant); + +/* Copy the JSON data into the target table. */ + +copy into house_sales + from @mystage/sales.json.gz; + + +create materialized view extjson_mv + as + select * from house_sales; + \ No newline at end of file diff --git a/databases/snowflake/copy-from-json/snowflake-data-linage-copy-to-json.png b/databases/snowflake/copy-from-json/snowflake-data-linage-copy-to-json.png new file mode 100644 index 0000000..d7431c5 Binary files /dev/null and b/databases/snowflake/copy-from-json/snowflake-data-linage-copy-to-json.png differ diff --git a/databases/snowflake/copy-into/copy-into-location.md b/databases/snowflake/copy-into/copy-into-location.md new file mode 100644 index 0000000..3620fbc --- /dev/null +++ b/databases/snowflake/copy-into/copy-into-location.md @@ -0,0 +1,117 @@ +## Snowflake copy into location +[Official SQL syntax link for copy into location](https://docs.snowflake.com/en/sql-reference/sql/copy-into-location.html) + +### internalStage + +#### SQL 1 +```sql +copy into @%orderstiny/result/data_ + from orderstiny file_format = (format_name ='myformat' compression='GZIP'); +``` + +The data lineage generated for this SQL: +``` +orderstiny(*) -> @%orderstiny(result/data_) +``` + +`@%orderstiny` is the stage. + +#### SQL 2 +```sql +copy into @my_stage/result/data_ from (select * from orderstiny) + file_format=(format_name='myformat' compression='gzip'); +``` + +The data lineage generated for this SQL: +``` +orderstiny(*) -> @my_stage(result/data_) +``` + +#### SQL 3, personal stage +```sql +copy into @~ from home_sales +file_format=(type=csv null_if = ('NULL', 'null') +empty_field_as_null=false); +``` + +The data lineage generated for this SQL: +``` +home_sales(*) -> @~(unknownPath) +``` + +### externalStage + +#### SQL 1 +```sql +copy into 's3://mybucket/unload/' + from mytable + storage_integration = myint + file_format = (format_name = my_csv_format); +``` + +The data lineage generated for this SQL: +``` +directory(s3://mybucket/unload/) -> mytable(*) +``` + + +#### SQL 2 +```sql +create or replace stage my_csv_stage + file_format = mycsvformat + url = 's3://snowflake-docs'; + +copy into mycsvtable + from @my_csv_stage/tutorials/dataloading/contacts1.csv + on_error = 'skip_file'; +``` + +The data lineage generated for this SQL: +``` +file(/tutorials/dataloading/contacts1.csv) - > @my_csv_stage(s3://snowflake-docs) -> mycsvtable(*) +``` + +### externalLocation +#### SQL 1 +```sql +copy into mytable + from 's3://mybucket/data/files' + storage_integration = myint + encryption=(master_key = 'eSxX0jzYfIamtnBKOEOwq80Au6NbSgPH5r4BDDwOaO8=') + file_format = (format_name = my_csv_format); +``` + +The data lineage generated for this SQL: +``` +directory('s3://mybucket/data/files') -> mytable(*) +``` + +### Partitioning Unloaded Rows to Parquet Files +```sql +create or replace table t1 ( + dt date, + ts time + ) +as + select to_date($1) + ,to_time($2) + from values + ('2020-01-28', '18:05') + ,('2020-01-28', '22:57') + ,('2020-01-28', null) + ,('2020-01-29', '02:15') +; + +copy into @%t1 + from t1 + partition by ('date=' || to_varchar(dt, 'YYYY-MM-DD') || '/hour=' || to_varchar(date_part(hour, ts))) -- Concatenate labels and column values to output meaningful filenames + file_format = (type=parquet) + max_file_size = 32000000 + header=true; +``` + +The data lineage generated for this SQL: +``` +t1(dt,ts) -> @%t1(unknownPath) +``` + diff --git a/databases/snowflake/copy-into/copy-into-table.md b/databases/snowflake/copy-into/copy-into-table.md new file mode 100644 index 0000000..4ddfab6 --- /dev/null +++ b/databases/snowflake/copy-into/copy-into-table.md @@ -0,0 +1,83 @@ +## Snowflake copy into table +[Official SQL syntax link for copy into table](https://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html#) + +### internalStage + +#### SQL 1 +```sql +copy into mytable +from @my_int_stage; +``` + +The data lineage generated for this SQL: +``` +@my_int_stage(unknownPath) -> mytable(*) +``` +`unknownPath` and `*` is added by the gudu SQLFlow, and those columns are marked as `source="system"`. + +#### SQL 2 +```sql +copy into mytable +file_format = (type = csv); +``` + +No data lineage will be generated for this SQL due to the lack of FROM clause. + +#### SQL 3 +```sql +copy into mytable from @~/staged +file_format = (format_name = 'mycsv'); +``` + +The data lineage generated for this SQL: +``` +@~(staged) -> mytable(*) +``` + +`@~` represents a person stage. + +### externalStage + +#### SQL 1 +```sql +copy into mycsvtable + from @my_ext_stage/tutorials/dataloading/contacts1.csv; +``` + +The data lineage generated for this SQL: +``` +file(/tutorials/dataloading/contacts1.csv) - > @my_ext_stage(unknownPath) -> mycsvtable(*) +``` + + +#### SQL 2 +```sql +create or replace stage my_csv_stage + file_format = mycsvformat + url = 's3://snowflake-docs'; + +copy into mycsvtable + from @my_csv_stage/tutorials/dataloading/contacts1.csv + on_error = 'skip_file'; +``` + +The data lineage generated for this SQL: +``` +file(/tutorials/dataloading/contacts1.csv) - > @my_csv_stage(s3://snowflake-docs) -> mycsvtable(*) +``` + +### externalLocation +#### SQL 1 +```sql +copy into mytable + from 's3://mybucket/data/files' + storage_integration = myint + encryption=(master_key = 'eSxX0jzYfIamtnBKOEOwq80Au6NbSgPH5r4BDDwOaO8=') + file_format = (format_name = my_csv_format); +``` + +The data lineage generated for this SQL: +``` +directory('s3://mybucket/data/files') -> mytable(*) +``` + diff --git a/databases/snowflake/copy-into/readme.md b/databases/snowflake/copy-into/readme.md new file mode 100644 index 0000000..99dcfc2 --- /dev/null +++ b/databases/snowflake/copy-into/readme.md @@ -0,0 +1,2 @@ +- [copy into table](copy-into-table.md) +- [copy into location](copy-into-location.md) \ No newline at end of file diff --git a/databases/snowflake/grabit-snowfalke-5-sqlflow-result.png b/databases/snowflake/grabit-snowfalke-5-sqlflow-result.png new file mode 100644 index 0000000..c4dd656 Binary files /dev/null and b/databases/snowflake/grabit-snowfalke-5-sqlflow-result.png differ diff --git a/databases/snowflake/grabit-snowflake-1.png b/databases/snowflake/grabit-snowflake-1.png new file mode 100644 index 0000000..89c4ef6 Binary files /dev/null and b/databases/snowflake/grabit-snowflake-1.png differ diff --git a/databases/snowflake/grabit-snowflake-2-database.png b/databases/snowflake/grabit-snowflake-2-database.png new file mode 100644 index 0000000..2bb9c03 Binary files /dev/null and b/databases/snowflake/grabit-snowflake-2-database.png differ diff --git a/databases/snowflake/grabit-snowflake-3-database-parameters.png b/databases/snowflake/grabit-snowflake-3-database-parameters.png new file mode 100644 index 0000000..3bf0271 Binary files /dev/null and b/databases/snowflake/grabit-snowflake-3-database-parameters.png differ diff --git a/databases/snowflake/grabit-snowflake-4-sqlflow.png b/databases/snowflake/grabit-snowflake-4-sqlflow.png new file mode 100644 index 0000000..f8cc3ed Binary files /dev/null and b/databases/snowflake/grabit-snowflake-4-sqlflow.png differ diff --git a/databases/snowflake/grabit-snowflake-6-data-lineage-result.png b/databases/snowflake/grabit-snowflake-6-data-lineage-result.png new file mode 100644 index 0000000..16699ed Binary files /dev/null and b/databases/snowflake/grabit-snowflake-6-data-lineage-result.png differ diff --git a/databases/snowflake/grabit-snowflake-command-line.md b/databases/snowflake/grabit-snowflake-command-line.md new file mode 100644 index 0000000..14815a5 --- /dev/null +++ b/databases/snowflake/grabit-snowflake-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from snowflake (Command Line Mode) +This article introduces how to discover the data lineage from snowflake scripts or the snowflake database and automatically update it. +So the business users and developers can see the snowflake data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a snowflake database +- Modify the `conf-template\snowflake-config-template` to meet your environment. + +Here is a sample config file: `snowflake-config` that grabs metadata from the remote snowflake database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseType":"snowflake", + "optionType":1, + "resultType":1, + "databaseServer":{ + "hostname":"snowflake ip address", + "port":"443", + "username":"snowflake user name", + "password":"your password here", + "database":"", + "extractedDbsSchemas":"DEMO_DB/PUBLIC", + "excludedDbsSchemas":"*/INFORMAITON_SCHEMA", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f snowflake-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/snowflake/grabit-snowflake-gui.md b/databases/snowflake/grabit-snowflake-gui.md new file mode 100644 index 0000000..9106f3a --- /dev/null +++ b/databases/snowflake/grabit-snowflake-gui.md @@ -0,0 +1,68 @@ +## Automated data lineage from Snowflake (GUI Mode) +This article introduces how to discover the data lineage from snowflake scripts or the snowflake database and automatically update it. +So the business users and developers can see the SQL Server data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a snowflake database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit snowflake UI 1](grabit-snowflake-1.png) + +- Select `snowflake` in the list + +![Grabit snowflake UI 2 database](grabit-snowflake-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in DEMO_DB/PUBLIC schema. + +![Grabit snowfalke UI 3 database parameters](grabit-snowflake-3-database-parameters.png) + +- note + +1.The `Database` parameter is must specified. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for all databases is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + + +- After grabbing the metadata from the snowflake database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit snowflake SQLFlow](grabit-snowflake-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit snowflake SQLFlow result](grabit-snowfalke-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit snowflake data lineage result](grabit-snowflake-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a Snowflake database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover snowflake data lineage in command line mode](grabit-snowflake-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a snowflake database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) diff --git a/databases/snowflake/readme.md b/databases/snowflake/readme.md new file mode 100644 index 0000000..2c0cd98 --- /dev/null +++ b/databases/snowflake/readme.md @@ -0,0 +1,325 @@ +# Snowflake column-level data lineage + +Discover and visualization lineage from Snowflake database and script. + +## Extract DDL from the database + +1、database: +```sql +show databases; +``` + +2、table, view： +```sql +select + '"' || t.table_catalog || '"' as dbName, + '"' || t.table_schema || '"' as schemaName, + '"' || t.table_name || '"' as tableName, + case when t.table_type = 'VIEW' then 'true' + when t.table_type = 'BASE TABLE' then 'false' + else 'false' + end as isView, + '"' || c.column_name || '"' as columnName, + c.data_type, + null as comments +from + "%s".information_schema.tables t, + "%s".information_schema.columns c +where + t.table_catalog = c.table_catalog + and t.table_schema = c.table_schema + and t.table_name = c.table_name + and upper(t.table_schema) not in ('INFORMATION_SCHEMA') +order by t.table_catalog, t.table_schema, t.table_name, c.ordinal_position; +``` +3、source code of the view +```sql +SHOW VIEWS IN %s.%s; +SELECT GET_DDL('VIEW', '%s.%s.%s'); +``` +4、source code of the procedure +```sql +SHOW PROCEDURES IN %s.%s; +SELECT GET_DDL('PROCEDURE', '%s.%s.%s'); +``` + +5、source code of the function: +```sql +SHOW FUNCTIONS IN %s.%s; +SELECT GET_DDL('FUNCTION', '%s.%s.%s'); +``` + +## a minimum list of permissions need to extract all DDL + +You must define a role that has access to the database of the DDL database you want to export and assign WAREHOUSE permissions to that role,If `SQLFlow_role` and `SQLFlow_user` are the roles and users you use when grabit connects to the Snowflake database, you need to do the following: + +1, First, you need to create a role, such as `SQLFlow_role` + +2, Next, you need to use the `ACCOUNTADMIN` role to assign the required database, schema, view, and table privileges to `SQLFlow_role` + +3, Next, create the user to access `SQLFlow_user` + +4, Finally, grant `SQLFlow_role` privileges to the `SQLFlow_user` + +```sql +create or replace role SQLFlow_role; + +use role accountadmin; + +// Grant privileges to use and select from your target warehouses / dbs / schemas / tables +grant operate, usage on warehouse to role SQLFlow_role; +grant usage on DATABASE to role SQLFlow_role; +grant usage on all schemas in database to role SQLFlow_role; +grant select on all tables in database to role SQLFlow_role; +grant select on all external tables in database to role SQLFlow_role; +grant select on all views in database to role SQLFlow_role; + +// Grant privileges for all future schemas and tables created in a warehouse +grant usage on future schemas in database "" to role SQLFlow_role; +grant select on future tables in database "" to role SQLFlow_role; + +// Create a new SQLFlow_user user and assign the SQLFlow role to it +create user SQLFlow_user display_name = 'SQLFlow' password='' default_role = SQLFlow_user default_warehouse = ''; + +// Grant the SQLFlow_role to the new SQLFlow_user user. +grant role SQLFlow_role to user SQLFlow_user; +``` + +This represents the bare minimum privileges required to extract databases, schemas, views, tables from Snowflake. + + +## enable extraction of table lineage + +If you plan to enable extraction of table lineage, via the include_table_lineage config flag, you'll also need to grant privileges to access the `Snowflake` Account Usage views. You can execute the following using the `ACCOUNTADMIN` role to do so: + +> You must define a role that has access to the SNOWFLAKE database,And assign WAREHOUSE permission to this role. + +grant privileges to a role, for example: + +````sql +use role accountadmin; +grant imported privileges on database snowflake to role SQLFlow_role; +```` + +## Using the grabit tool +1. [GUI Mode](grabit-snowflake-gui.md) +2. [Command Line](grabit-snowflake-command-line.md) + +### Parameters used in grabit tool + +#### hostname + +The IP of the database server that the grabit connects. + +#### port + +The port number of the database server that the grabit connect. + +#### username + +The database user used to login to the database. + +#### password + +The password of the database user. + +note: the passwords can be encrypted using tools [Encrypted password](#Encrypted password), using encrypted passwords more secure. + +#### privateKeyFile + +Use a private key to connect. + +#### privateKeyFilePwd + +Generate the password for the private key. + +#### database + +The name of the database instance to which it is connected, it is optional. + +#### extractedDbsSchemas + +List of databases and schemas to extract, separated by +commas, which are to be provided in the format database/schema; +Or blank to extract all databases. +`database1/schema1,database2/schema2,database3` or `database1.schema1,database2.schema2,database3` +When parameter `database` is filled in, this parameter is considered a schema. +And support wildcard characters such as `database1/*`,`*/schema`,`*/*`. + +for example: +````json +extractedDbsSchemas: "MY/ADMIN" +```` + + +#### excludedDbsSchemas + +This parameters works under the resultset filtered by `extractedDbsSchemas`. +List of databases and schemas to exclude from extraction, separated by commas +`database1/schema1,database2` or `database1.schema1,database2` +When parameter `database` is filled in, this parameter is considered a schema. +And support wildcard characters such as `database1/*`,`*/schema`,`*/*`. + +for example: +````json +excludedDbsSchemas: "MY/*" +```` + +#### extractedStoredProcedures + +A list of stored procedures under the specified database and schema to extract, separated by +commas, which are to be provided in the format database.schema.procedureName or schema.procedureName; +Or blank to extract all databases, support expression. +`database1.schema1.procedureName1,database2.schema2.procedureName2,database3.schema3,database4` or `database1/schema1/procedureName1,database2/schema2` + +for example: + +````json +extractedStoredProcedures: "database.scott.vEmp*" +```` + +or + +````json +extractedStoredProcedures: "database.scott" +```` + +#### extractedViews + +A list of stored views under the specified database and schema to extract, separated by +commas, which are to be provided in the format database.schema.viewName or schema.viewName. +Or blank to extract all databases, support expression. +`database1.schema1.procedureName1,database2.schema2.procedureName2,database3.schema3,database4` or `database1/schema1/procedureName1,database2/schema2` + +for example: + +````json +extractedViews: "database.scott.vEmp*" +```` + +or + +````json +extractedViews: "database.scott" +```` + +#### enableQueryHistory + +Fetch SQL queries from the query history if set to `true` default is false. + +#### Extract from the query history +This is the SQL query used to get query from the snowflake query history,We can extract data from the last year. +```sql +SELECT +* +FROM +"SNOWFLAKE"."ACCOUNT_USAGE"."QUERY_HISTORY" +WHERE +dateadd('mins', +-%s, +current_timestamp()) <= start_time ORDER BY start_time; + +``` + +#### permission needs to extract queries from query history + +You must define a role that has access to the `SNOWFLAKE` database,And assign `WAREHOUSE` permission to this role. please ref to: [a minimum list of permissions need to extract all DDL](#a-minimum-list-of-permissions-need-to-extract-all-DDL) + +#### queryHistoryBlockOfTimeInMinutes + +When `enableQueryHistory:true`, the interval at which the SQL query was extracted in the query History,default is `30` minutes. + +#### queryHistorySqlType + +When `enableQueryHistory:true`, the DML type of SQL is extracted from the query History. +When empty, all types are extracted, and when multiple types are specified, a comma separates them, such as `SELECT,UPDATE,MERGE`. +Currently only the snowflake database supports this parameter,support types are **SHOW,SELECT,INSERT,UPDATE,DELETE,MERGE,CREATE TABLE, CREATE VIEW, CREATE PROCEDURE, CREATE FUNCTION**. + +for example: + +````json +queryHistorySqlType: "SELECT,DELETE" +```` + +#### snowflakeDefaultRole + +This value represents the role of the snowflake database. + +#### sqlsourceTableName + +table name: **query_table** + +| query_name | query_source | +| ---------- | ----------------------------------- | +| query1 | create view v1 as select f1 from t1 | +| query2 | create view v2 as select f2 from t2 | +| query3 | create view v3 as select f3 from t3 | + +If you save SQL queries in a specific table, one SQL query per row. + +Let's say: `query_table.query_source` store the source code of the query. +We can use this query to fetch all SQL queries in this table: + +```sql +select query_name as queryName, query_source as querySource from query_table +``` + +By setting the value of `sqlsourceTableName` and `sqlsourceColumnQuerySource`,`sqlsourceColumnQueryName` +grabit can fetch all SQL queries in this table and send it to the SQLFlow to analzye the lineage. + +In this example, +```json +"sqlsourceTableName":"query_table" +"sqlsourceColumnQuerySource":"query_source" +"sqlsourceColumnQueryName":"query_name" +``` + +Please leave `sqlsource_table_name` empty if you don't fetch SQL queries from a specific table. + +#### sqlsourceColumnQuerySource +In the above sample: +```json +"sqlsourceColumnQuerySource":"query_source" +``` + +#### sqlsourceColumnQueryName +```json +"sqlsourceColumnQueryName":"query_name" +``` +This parameter is optional, you don't need to speicify a query name column if it doesn't exist in the table. + + +**eg configuration file:** +````json +{ + "databaseServer":{ + "hostname":"127.0.0.1", + "port":"433", + "username":"USERNAME", + "password":"PASSWORD", + "privateKeyFile":"", + "privateKeyFilePwd":"", + "database":"", + "extractedDbsSchemas":"MY/dbo", + "excludedDbsSchemas":"", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":true, + "queryHistoryBlockOfTimeInMinutes":30, + "snowflakeDefaultRole":"", + "queryHistorySqlType":"", + "sqlsourceTableName":"", + "sqlsourceColumnQuerySource":"", + "sqlsourceColumnQueryName":"" + }, + "SQLFlowServer":{ + "server":"http:127.0.0.1", + "serverPort":"8081", + "userId":"gudu|0123456789", + "userSecret":"" + }, + "SQLScriptSource":"database", + "lineageReturnFormat":"json", + "databaseType":"snowflake" +} +```` diff --git a/databases/sparksql/readme.md b/databases/sparksql/readme.md new file mode 100644 index 0000000..a9ef233 --- /dev/null +++ b/databases/sparksql/readme.md @@ -0,0 +1,97 @@ +## SparkSQL column-level data lineage + +Discover and visualization lineage from SparkSQL script. +SQLFlow supports alls SQL statements of SparkSQL 3. Below are some examples that +illustrate how SQLFlow works. You may try your SparkSQL query to get the lineage +using the [SQLFlow Cloud](https://sqlflow.gudusoft.com). + +### INSERT OVERWRITE DIRECTORY +```sql +INSERT OVERWRITE DIRECTORY 's3:///bucket/path/to/report' + USING parquet + OPTIONS (col1 1, col2 'sum') + SELECT bar.my_flag,sum(foo.amount) as amount_sum + FROM mydb.foo foo + left join mydb.bar bar + on foo.bar_fk = bar.pk + group by bar.my_flag; +``` + +The lineage diagram: + +[![sparksql insert overwrite directory](/images/sparksql-insert-overwrite-directory.png)](https://sqlflow.gudusoft.com) + + +### Pivot clause +```sql +CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING); + +SELECT * FROM person + PIVOT ( + SUM(age) AS a, AVG(class) AS c + FOR name IN ('John' AS john, 'Mike' AS mike) + ); +``` + +The lineage diagram: + +[![sparksql pivot clause](/images/sparksql-pivot-clause.png)](https://sqlflow.gudusoft.com) + + +### [Try your SparkSQL queries using the SQLFlow Cloud Version](https://sqlflow.gudusoft.com) + +### Programmatically using [Restful APIs](/api) or [SDKs](https://www.gudusoft.com/sqlflow-java-library-2/) to get lineage in CSV, JSON, Graphml format. + +### SparkSQL sapmle SQLs for reference +```sql +INSERT OVERWRITE DIRECTORY 's3:///bucket/path/to/report' + USING parquet + OPTIONS (col1 1, col2 'sum') + SELECT bar.my_flag,sum(foo.amount) as amount_sum + FROM mydb.foo foo + left join mydb.bar bar + on foo.bar_fk = bar.pk + group by bar.my_flag; + +INSERT OVERWRITE DIRECTORY + USING parquet + OPTIONS ('path' 's3:///bucket/path/to/report', col1 1, col2 'sum') + SELECT bar.my_flag,sum(foo.amount) as amount_sum + FROM mydb.foo foo + left join mydb.bar bar + on foo.bar_fk = bar.pk + group by bar.my_flag; + + +create schema mydb; + +create table mydb.bar( + pk int, + my_flag int +); + + +create table mydb.foo( + bar_fk int, + amount int +); + +insert into mydb.bar(pk,my_flag) values(1, 100); +insert into mydb.bar(pk,my_flag) values(2, 200); +insert into mydb.bar(pk,my_flag) values(3, 300); +insert into mydb.bar(pk,my_flag) values(4, 400); +insert into mydb.foo(bar_fk,amount) values(1, 10); +insert into mydb.foo(bar_fk,amount) values(1, 20); +insert into mydb.foo(bar_fk,amount) values(2, 200); +insert into mydb.foo(bar_fk,amount) values(2, 300); +insert into mydb.foo(bar_fk,amount) values(3, 250); +insert into mydb.foo(bar_fk,amount) values(4, 350); + + +SELECT bar.my_flag,sum(foo.amount) as amount_sum +FROM mydb.foo foo +left join mydb.bar bar +on foo.bar_fk = bar.pk +group by bar.my_flag +; +``` \ No newline at end of file diff --git a/databases/sql-server/grabit-sql-server-1.png b/databases/sql-server/grabit-sql-server-1.png new file mode 100644 index 0000000..89c4ef6 Binary files /dev/null and b/databases/sql-server/grabit-sql-server-1.png differ diff --git a/databases/sql-server/grabit-sql-server-2-database.png b/databases/sql-server/grabit-sql-server-2-database.png new file mode 100644 index 0000000..4b803b5 Binary files /dev/null and b/databases/sql-server/grabit-sql-server-2-database.png differ diff --git a/databases/sql-server/grabit-sql-server-3-database-parameters.png b/databases/sql-server/grabit-sql-server-3-database-parameters.png new file mode 100644 index 0000000..851cb83 Binary files /dev/null and b/databases/sql-server/grabit-sql-server-3-database-parameters.png differ diff --git a/databases/sql-server/grabit-sql-server-4-sqlflow.png b/databases/sql-server/grabit-sql-server-4-sqlflow.png new file mode 100644 index 0000000..f8cc3ed Binary files /dev/null and b/databases/sql-server/grabit-sql-server-4-sqlflow.png differ diff --git a/databases/sql-server/grabit-sql-server-5-sqlflow-result.png b/databases/sql-server/grabit-sql-server-5-sqlflow-result.png new file mode 100644 index 0000000..6264b6b Binary files /dev/null and b/databases/sql-server/grabit-sql-server-5-sqlflow-result.png differ diff --git a/databases/sql-server/grabit-sql-server-6-data-lineage-result.png b/databases/sql-server/grabit-sql-server-6-data-lineage-result.png new file mode 100644 index 0000000..9189476 Binary files /dev/null and b/databases/sql-server/grabit-sql-server-6-data-lineage-result.png differ diff --git a/databases/sql-server/grabit-sql-server-command-line.md b/databases/sql-server/grabit-sql-server-command-line.md new file mode 100644 index 0000000..be92c02 --- /dev/null +++ b/databases/sql-server/grabit-sql-server-command-line.md @@ -0,0 +1,70 @@ +## Automated data lineage from SQL Server (Command Line Mode) +This article introduces how to discover the data lineage from SQL Server scripts or the SQL Server database and automatically update it. +So the business users and developers can see the SQL Server data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a SQL Server database +- Modify the `conf-template\sqlserver-config-template` to meet your environment. + +Here is a sample config file: `sqlserver-config` that grabs metadata from a local SQL Server database +and sends the metadata to the SQLFlow Cloud to discover the data lineage. + +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + + +```json +{ + "databaseServer":{ + "hostname":"sql server ip address", + "port":"1433", + "username":"sql server user name", + "password":"your password here", + "database":"", + "extractedDbsSchemas":"", + "excludedDbsSchemas":"master/dbo,master/sys,master/INFORMAITON_SCHEMA,msdb/dbo,tempdb/dbo,tempdb/sys,model/dbo", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your sqlflow premium account id", + "userSecret":"your sqlflow premium account secret code" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "optionType":1, + "resultType":1, + "databaseType":"sqlserver", + "isUploadNeo4j":0 +} +``` + +- Run grabit command-line tool, you may find the grabit.log under the logs directory. +``` +./start.sh /f sqlserver-config +``` + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +- You may save the data lineage in JSON/CSV/GRAPHML format. + + The file will be saved under `data\datalineage` directory. + +- Run the grabit at a scheduled time + + [Please check the instructions here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#run-the-grabit-at-a-scheduled-time) + diff --git a/databases/sql-server/readme.md b/databases/sql-server/readme.md new file mode 100644 index 0000000..6f176d5 --- /dev/null +++ b/databases/sql-server/readme.md @@ -0,0 +1,72 @@ +## Automated data lineage from SQL Server (GUI Mode) +This article introduces how to discover the data lineage from SQL Server scripts or the SQL Server database and automatically update it. +So the business users and developers can see the SQL Server data lineage graph instantly. + +### Software used in this solution +- [SQLFlow Cloud](https://sqlflow.gudusoft.com) Or [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + +### Discover data lineage in a SQL Server database +- After [start up the grabit tool](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#running-the-grabit-tool), this is the first UI. +Click the `database` button. + +![Grabit SQL Server UI 1](grabit-sql-server-1.png) + +- Select `sql server` in the list + +![Grabit SQL Server UI 2 database](grabit-sql-server-2-database.png) + +- Set the database parameters. In this example, we only discover the data lineage in AdventureWorksDW2019/dbo schema. + +![Grabit SQL Server UI 3 database parameters](grabit-sql-server-3-database-parameters.png) + +- note + +1.The `Database` parameter is optional. + +2.When the `ExtractedDBSSchemas` and `ExcludedDBSSchemas` parameters are null, all data for all databases is retrieved by default. + +3.If you just want to get all the data in the specified database, you can use the following configuration to achieve this: `ExtractedDBSSchemas: db/*`. + +- After grabbing the metadata from the SQL Server database, connect to the SQLFlow server. +It would help if you had [a premium account](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) to access the SQLFlow Cloud. + +![Grabit SQL Server SQLFlow](grabit-sql-server-4-sqlflow.png) + +- Submit the database metadata to the SQLFlow server and get the data lineage +![Grabit SQL Server SQLFlow result](grabit-sql-server-5-sqlflow-result.png) + +- Check out the diagram via this url: [https://sqlflow.gudusoft.com/#/job/latest](https://sqlflow.gudusoft.com/#/job/latest) + +![Grabit SQL Server data lineage result](grabit-sql-server-6-data-lineage-result.png) + +- You may save the data lineage in JSON/CSV/GRAPHML format + +The file will be saved under `data\datalineage` directory. + +### Further information +This tutorial illustrates how to discover the data lineage of a SQL Server database in the grabit UI mode, +If you like to automated the data lineage discovery, you may use the Grabit command line mode. + +- [Discover SQL Server data lineage in command line mode](grabit-sql-server-command-line.md) + + +This tutorial illustrates how to discover the data lineage of a SQL Server database by submitting the database +metadata to the SQLFlow Cloud version, You may set up the [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +on your server to secure your information. + +For more options of the grabit tool, please check this page. +- [Grabit tool readme](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) + +The completed guide of SQLFlow UI +- [How to use SQLFlow](https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow_guide.md) + + + +### Know-How +![sqlflow-automated-data-lineage](/images/sqlflow-overview-grabit.png "SQLFlow automated data lineage") diff --git a/databases/sql-server/sql-server-data-lineage-with-metadata.png b/databases/sql-server/sql-server-data-lineage-with-metadata.png new file mode 100644 index 0000000..c3b5a51 Binary files /dev/null and b/databases/sql-server/sql-server-data-lineage-with-metadata.png differ diff --git a/databases/sql-server/sql-server-data-lineage-without-metadata.png b/databases/sql-server/sql-server-data-lineage-without-metadata.png new file mode 100644 index 0000000..9432f5f Binary files /dev/null and b/databases/sql-server/sql-server-data-lineage-without-metadata.png differ diff --git a/databases/sql-server/sql-server-resolve-ambiguous-column-problem.md b/databases/sql-server/sql-server-resolve-ambiguous-column-problem.md new file mode 100644 index 0000000..08f158c --- /dev/null +++ b/databases/sql-server/sql-server-resolve-ambiguous-column-problem.md @@ -0,0 +1,71 @@ +## Discover data lineage in a SQL script file with the metadata from the SQL Server database + +Let's say we like to get data linage from this sample SQL: + +```SQL +use AdventureWorksDW2019 + +SELECT e.firstName, e.SalesTerritoryKey, SalesTerritoryRegion +FROM dbo.DimEmployee e +left join dbo.DimSalesTerritory d +on e.SalesTerritoryKey = d.SalesTerritoryKey +``` + +As you can see, due to the lack of metadata information, +column `SalesTerritoryRegion` is treated as a column of table: `dbo.DimEmployee`, which is not correct. +This kind of error is due to the lack of the metadata from the database. + +![data lineage without metadata](./sql-server-data-lineage-without-metadata.png "data lineage without metadata") + + +This is the data lineage with support of the metadata from the database. +![data lineage with metadata](sql-server-data-lineage-with-metadata.png "data lineage with metadata") + +Here is the schema information from the database, `dbo.DimSalesTerritory` includes the column `SalesTerritoryRegion` + +![sql-server-schema-column](sql-server-schema-info.png) + +In order to resolve ambiguous column problem, grabit tool will connect to the database to grab the metadata and +then resolve this problem. + +Here is the configuration used by the grabit tool: +```json +{ + "databaseType":"sqlserver", + "optionType":4, + "resultType":1, + "SQLInSingleFile":{ + "filePath":"/home/ubuntu/demo.sql" + }, + "enableGetMetadataInJSONFromDatabase":1, + + + "databaseServer":{ + "hostname":"115.159.225.38", + "port":"1433", + "username":"sa", + "password":"your password here", + "sid":"", + "extractedDbsSchemas":"AdventureWorksDW2019/dbo", + "excludedDbsSchemas":"master/dbo,master/sys,master/INFORMAITON_SCHEMA,msdb/dbo,tempdb/dbo,tempdb/sys,model/dbo", + "extractedStoredProcedures":"", + "extractedViews":"", + "enableQueryHistory":false, + "queryHistoryBlockOfTimeInMinutes":30 + }, + "SQLFlowServer":{ + "server":"http://server.sqlflow.cn", + "serverPort":"8081", + "userId":"gudu|0123456789", + "userSecret":"" + }, + "neo4jConnection":{ + "url":"", + "username":"", + "password":"" + }, + "isUploadNeo4j":0 +} +``` + + diff --git a/databases/sql-server/sql-server-schema-info.png b/databases/sql-server/sql-server-schema-info.png new file mode 100644 index 0000000..d5286fd Binary files /dev/null and b/databases/sql-server/sql-server-schema-info.png differ diff --git a/dbobjects_relationship.md b/dbobjects_relationship.md new file mode 100644 index 0000000..384af3c --- /dev/null +++ b/dbobjects_relationship.md @@ -0,0 +1,278 @@ +## The basic concept of the dataflow + +### The main relation type in dataflow +1. column to column dataflow, the data of target column is coming from the source column(fdd). +2. column to resultset(mainly select list), the number of row in the resultset is impacted by the source column(fdr). +3. resultset to resultset, the number of row in a source table in the from clause impact the number of row in the target select list.(fdr) + +### Analyze dataflow in the different SQL elements - part 1 +1. select list +2. where clause +3. function (case expression) +4. group by(aggregate function) +5. from clause +6. handle of select * (Not finished yet) + +#### 1. select list +```sql +SELECT a.empName "eName" +FROM scott.emp a +``` +the data of target column `"eName"` comes from `scott.emp.empName`, so we have a dataflow relation like this: +``` +scott.emp.empName -> fdd -> "eName" +``` +the result generated by the select list called: `resultset` likes a virtual table includes columns and rows. + +#### 2. where clause +```sql +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` +The total number of row in the select list is impacted by the value of column `sal` in the where clause. So we have a dataflow relation like this: +``` +sal -> fdr -> resultset.PseudoRows +``` + +As you can see, we introduced a new pseudo column: `PseudoRows` to represents the number of rows in the resultset. + +#### 3. function +During the dataflow analyzing, `function` plays a key role. It accepts arguments which usually is column and generate resultset which maybe a scalar value or a set value. +```sql +select round(salary) as sal from scott.emp +``` + +The relation of the `round` function in the above SQL : +``` +scott.emp.salary -> fdd -> round(salary) -> fdd -> sal +``` + +#### 4. group by and aggregate function +```sql +SELECT deptno, COUNT() num_emp, SUM(SAL) sal_sum +FROM scott.emp +Where city = 'NYC' +GROUP BY deptno +``` + +##### 4.1 + +since `SUM()` is an aggregate function, so `deptno` column in the group by clause will be treated as an implict argument of the `SUM()` function. +However, `deptno` column doesn't directly contribute the value to the `SUM()` function as column `SAL` does, So, the relation type is `fdr`: +``` +scott.emp.deptno -> fdr -> SUM(SAL) -> fdd -> sal_sum +``` + +the columns in the having clause have the same relation as the columns in the group by clause as mentioned above. + +##### 4.2 +The value of `SUM()` function also effected by the total rows of the table `scott.emp`, so, there is a relation like this: +``` +scott.emp.pseudoRows -> fdr -> SUM(SAL) -> fdd -> sal_sum +``` + +The above rules apply to all aggregation functions, such as the `count()` function in the SQL. + +#### 5. From clause +If the resultset of a subquery or CTE is used in the from clause of the upper-level statement, then the impact of the lower level resultset will be transferred to the upper-level. +```sql +WITH + cteReports (EmpID, FirstName, LastName, MgrID, EmpLevel) + AS + ( + SELECT EmployeeID, FirstName, LastName, ManagerID, 1 -- resultset1 + FROM Employees + WHERE ManagerID IS NULL + ) +SELECT + FirstName + ' ' + LastName AS FullName, EmpLevel -- resultset2 +FROM cteReports +``` + +In the CTE, there is an impact relation: +``` +Employees.ManagerID -> fdr -> resultset1.pseudoRows +``` +Since `cteReports` is used in the from clause of the upper-level statement, then the impact will carry on like this: +``` +Employees.ManagerID -> fdr -> resultset1.pseudoRows -> fdd -> resultset2.pseudoRows +``` + +If we choose to ignore the intermediate resultset, the end to end dataflow is : +``` +Employees.ManagerID -> fdr -> resultset2.pseudoRows +``` + + +### Handle the dataflow chain +Every relation in the SQL is picked up by the tool, and connected together to show the whole dataflow chain. +Sometimes, we only need to see the end to end relation and ignore all the intermediate relations. + +If we need to convert a fully chained dataflow to an `end to end` dataflow, we may consider the following rules: + +1. A single dataflow chain with the mixed relation types: fdd and fdr. + ``` + A -> fdd -> B -> fdr -> C -> fdd -> D + ``` + the rule is: if any `fdr` relation appears in the chain, the relation from `A -> D` will be consider as type of `fdr`, otherwise, the final relation is `fdd` for the end to end relation of `A -> D`. + +2. If there are multiple chains from `A -> D` + ``` + A -> fdd -> B1 -> fdr -> C1 -> fdd -> D + A -> fdd -> B2 -> fdr -> C1 -> fdd -> D + A -> fdd -> B3 -> fdd -> C3 -> fdd -> D + ``` + The final relation should choose the `fdd` chain if any. + +### analyze dataflow in the different SQL elements - part 2 +#### 1. case expression + +```sql + select + case when a.kamut=1 and b.teur IS null + then 'no locks' + when a.kamut=1 + then b.teur + else 'locks' + end teur + from tbl a left join TT b on (a.key=b.key) +``` +During the analyzing of dataflow, case expression is treated as a function. The column used inside the case expression will be treated like the arguments of a function. +So for the above SQL, the following relation is discovered: +``` +tbl.kamut -> fdd -> teur +TT.teur -> fdd -> teur +``` + +#### 2. join condition + +Columns in the join condition also effect the number of row in the resultset of the select list just like column in the where clause do. +So, the following relation will be discoverd in the above SQL. +``` +tbl.key -> fdr -> resultset.PseudoRows +TT.key -> fdr -> resultset.PseudoRows +``` + +#### 3. create view +```sql +create view vEmp(eName) as +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` +From this query, you will see how the column `sal` in where clause impact the number of rows in the top level view `vEmp`. +``` +scott.emp.sal -> fdr -> resultset1.PseudoRows -> fdr -> vEmp.PseudoRows +``` + +So, from an end to end point of view, there will be a `fdr` relation between column `sal` and view `vEmp` like this: +``` +scott.emp.sal -> fdr -> vEmp.PseudoRows +``` + +#### 4. rename/swap table +```sql +alter table t2 rename to t3; +``` +We also use `PseudoRows` to represent the relation when rename a table, the relation type is `fdd`. +``` +t2.PseudoRows -> fdd -> t3.PseudoRows +``` + +#### 5. create external table (snowflake) +```sql +create or replace stage exttable_part_stage + url='s3://load/encrypted_files/' + credentials=(aws_key_id='1a2b3c' aws_secret_key='4x5y6z') + encryption=(type='AWS_SSE_KMS' kms_key_id = 'aws/key'); + +create external table exttable_part( + date_part date as to_date(split_part(metadata$filename, '/', 3) + || '/' || split_part(metadata$filename, '/', 4) + || '/' || split_part(metadata$filename, '/', 5), 'YYYY/MM/DD'), + timestamp bigint as (value:timestamp::bigint), + col2 varchar as (value:col2::varchar)) + partition by (date_part) + location=@exttable_part_stage/logs/ + auto_refresh = true + file_format = (type = parquet); +``` + +The data of the external table `exttable_part` comes from the stage: `exttable_part_stage` +``` +exttable_part_stage (url='s3://load/encrypted_files/') -> fdd -> exttable_part(date_part,timestamp,col2) +``` + +#### 6. create external table (bigquery) +```sql +CREATE EXTERNAL TABLE dataset.CsvTable OPTIONS ( + format = 'CSV', + uris = ['gs://bucket/path1.csv', 'gs://bucket/path2.csv'] +); +``` + +The data of the external table `dataset.CsvTable` comes from the csv file: `gs://bucket/path1.csv, gs://bucket/path2.csv` +``` +file (uri='gs://bucket/path1.csv') -> fdd -> dataset.CsvTable +file (uri='gs://bucket/path2.csv') -> fdd -> dataset.CsvTable +``` + +![bigquery create external table](/images/bigquery-create-external-table.png) + +#### 7. build data lineage for the foreign key in the create table statement. +```sql +CREATE TABLE masteTable +( + masterColumn varchar(3) Primary Key, +); + + +CREATE TABLE foreignTable +( + foreignColumn1 varchar(3) NOT NULL , + foreignColumn2 varchar(3) NOT NULL + FOREIGN KEY (foreignColumn1) REFERENCES masteTable(masterColumn), + FOREIGN KEY (foreignColumn2) REFERENCES masteTable(masterColumn) +) +``` + +The data flow is: +``` +masteTable.masterColumn -> fdd -> foreignTable.foreignColumn1 +masteTable.masterColumn -> fdd -> foreignTable.foreignColumn2 +``` + +#### 8, Hive load data +```sql +LOAD DATA LOCAL INPATH /tmp/pv_2008-06-08_us.txt INTO TABLE page_view PARTITION(date='2008-06-08', country='US') +``` + +The data flow is: +``` +file (/tmp/pv_2008-06-08_us.txt) -> fdd -> page_view(date,country) +``` +![hive load data](/images/hive-load-data.png) + +#### 9, Hive INSERT OVERWRITE [LOCAL] DIRECTORY +```sql +INSERT OVERWRITE LOCAL DIRECTORY '/tmp/pv_gender_sum' +SELECT pv_gender_sum.* +FROM pv_gender_sum; +``` + +The data flow is: +``` +pv_gender_sum(*) -> fdd -> file ('/tmp/pv_gender_sum') +``` + + +### The meaning of the letter in fdd, fdr + +The meaning of the letter in fdd, fdr. f: dataflow, d: data value, r: record set. + +The first letter is always f，the second letter represents the source column，the third letter represents the target column, the fourth is reserved. + +* fdd: data of the source column will used in the target column +* fdr: data of the source column will impact the number of the resultset in the select list, or will impact the result value of an anggreate function. + diff --git a/dbobjects_relationship_v1.md b/dbobjects_relationship_v1.md new file mode 100644 index 0000000..c7574b3 --- /dev/null +++ b/dbobjects_relationship_v1.md @@ -0,0 +1,196 @@ +Relationships in the data lineage generated by the [SQLFlow](https://gudusoft.com/sqlflow/#/). + +- [column relationship](#type-of-the-column-relationships) +- [table relationship](#type-of-the-table-relationships) + +## Type of the column relationships + +* **fdd**, the value of target column is come from source column, such as: t = s + 2 + + You may check `effectType` to see how the target column is changed. + + - `effectType = select`, the source data is from select. + - `effectType = insert`, the source data is from insert. + - `effectType = update`, the source data is from update. + - `effectType = merge_update`, the source data is merge update. + - `effectType = merege_insert`, the source data is from merege insert. + - `effectType = create_view`, the source data is from create view. + +* **fddi**, the value of the target column is not derived from the source column directly, but it is affected by the source column. + However, it's difficult to determine this kind of relation, take this syntax for example: t=fx(s). + so the relationship of the source and target column is `fdd` in a function call unless we know clearly that `t` will not + include value from `s`. + + In this sample SQL, the relationship between `teur` and `kamut` is fddi. + + ```sql + select + case when a.kamut=1 and b.teur IS null + then 'no locks' + when a.kamut=1 + then b.teur + else 'locks' + end teur + from tbl a left join TT b on (a.key=b.key) + ``` + + + +* **fdr**, the value of target column affected by the row number of the source table. It always happens when the aggregate function is used. + the source column may be appeared in the where，group by clause. This kind of relationship may also appear between the target column and the source table. + + +* **frd**, the row number of target column is affected by the value of source column. The source column usually appears in the where clause. + + +The meaning of the letter in `fdd`, `fddi`, `fdr`, `frd`, f: dataflow, d: data value, r: record set. + +The first letter is always f，the second letter represents the source column，the third letter represents the target column, the fourth is reserved. + + +### fdd +The value of the target column is derived from the source column directly. +```sql +create view vEmp(eName) as +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` + +`vEmp.eName` <- fdd <- `"eName"` <- fdd <- `scott.emp.empName` + +### fdr +The value of the target column is influenced by a source table itself, for example by the number of records. +This is caused by the use of aggregate function in the query. +```sql +create view vSal as +SELECT a.deptno "Department", + a.num_emp/b.total_count "Employees", + a.sal_sum/b.total_sal "Salary" +FROM + (SELECT deptno, COUNT() num_emp, SUM(SAL) sal_sum + FROM scott.emp + Where city = 'NYC' + GROUP BY deptno) a, + (SELECT COUNT() total_count, SUM(sal) total_sal + FROM scott.emp + Where city = 'NYC') b +``` + +`vSal.Salary` depends on the record number of table: `scott.emp`. +This is due to the use of aggregate function `COUNT()`, `SUM()` in the query, +`vSal.Salary` also depends on the `scott.emp.deptno` column which is used in the +group by clause. + +The `city` column in the where clause also determines the value of `vSal.Salary`. + +The chain of the dataflow is: + +`vSal.Salary` <- fdd <- `query_resultset.Salary` + +`query_resultset.Salary` <- fdd <- `sal_sum` + +`query_resultset.Salary` <- fdd <- `total_sal` + +`sal_sum` <- fdd <- `scott.emp.SAL` + +`sal_sum` <- fdr <- `scott.emp` + +`sal_sum` <- fdr <- `scott.emp.city` + +`sal_sum` <- fdr <- `scott.emp.deptno` + +`total_sal` <- fdd <- `scott.emp.sal` + +`total_sal` <- fdr <- `scott.emp` + +`total_sal` <- fdr <- `scott.emp.city` + + +### frd +Some of the columns in source tables such as WHERE clause do not influence the value of target columns +but are crucial for the selected row set, so they are also saved for impact analyses, +with relationship to the target columns. +```sql +create view vEmp(eName) as +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` +The value of `vEmp.eName` doesn’t depends on `scott.emp.sal`, +but the number of records in the `vEmp` depends on the `scott.emp.sal`, +so this tool record this kind of relationship as well + +`vEmp.eName` <- fdd <- `query_resultset."eName"` <- fdd <- `scott.emp.empName` + +`vEmp.eName` <- fdd <- `query_resultset."eName"` <- frd <- `scott.emp.sal` + +### fddi +The value of the target column depends on the value of the source column, but not come from the source column. +```sql +select + case when a.kamut=1 and b.teur IS null + then 'no locks' + when a.kamut=1 + then b.teur + else 'locks' + end teur +from tbl a left join TT b on (a.key=b.key) +``` + +The value of the select result: `teur` depends on the source column `tbl.kamut` +in the case expression, although it values is not derived from `tbl.kamut` directly. + +`query_result.teur` <- fddi <- `tbl.kamut` + +`query_result.teur` <- frd <- `tbl.key` + +`query_result.teur` <- frd <- `TT.key` + + +## Type of the table relationships +During ETL, It's typically to work with the staging tables and later rename them to the prod table names, or even better, swap the prod and staging tables in a single operation. +So there is a need to understand this kind of transition. + +```sql +create table t2 as select c from t1; +alter table t2 rename to t3; +create table t4 as select * from t3; + +OR + +create table t2 as select c from t1; +alter table t2 swap with t3; +create table t4 as select * from t3; +``` + +We also use `fdd` to reprenst those 2 relations. + +```sql +alter table t2 rename to t3; +``` +This relation should be built in the dataflow output. +`t2` -> fdd -> `t3` + +```xml + + + + +``` + + +```sql +alter table t2 swap with t3; +``` +This relation should be built in the dataflow output. +`t2` -> fdd -> `t3` + +```xml + + + + +``` + + diff --git a/demos/top-level-select-list/data-lineage-json-by-python.json b/demos/top-level-select-list/data-lineage-json-by-python.json new file mode 100644 index 0000000..8a84889 --- /dev/null +++ b/demos/top-level-select-list/data-lineage-json-by-python.json @@ -0,0 +1,480 @@ +{"code": 200, + "data": {"graph": {"elements": {"edges": [{"id": "e0", + "sourceId": "n2::n0", + "targetId": "n1::n0"}, + {"id": "e1", + "sourceId": "n3::n0", + "targetId": "n1::n3"}, + {"id": "e2", + "sourceId": "n0::n0", + "targetId": "n1::n2"}, + {"id": "e3", + "sourceId": "n0::n1", + "targetId": "n1::n1"}], + "tables": [{"columns": [{"height": 16.0, + "id": "n0::n0", + "label": {"content": "department_id", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 107.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 1.0, + "y": -46.00015}, + {"height": 16.0, + "id": "n0::n1", + "label": {"content": "last_name", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 1.0, + "y": -30.000149}], + "height": 57.96875, + "id": "n0", + "label": {"content": "employees", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162.0, + "x": 0.0, + "y": 0.0}, + "width": 162.0, + "x": 0.0, + "y": -67.9689}, + {"columns": [{"height": 16.0, + "id": "n1::n0", + "label": {"content": "remote", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 233.0, + "y": -130.72316}, + {"height": 16.0, + "id": "n1::n1", + "label": {"content": "LAST_NAME", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 86.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 233.0, + "y": -114.72317}, + {"height": 16.0, + "id": "n1::n2", + "label": {"content": "DEPARTMENT_ID", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 116.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 233.0, + "y": -98.72317}, + {"height": 16.0, + "id": "n1::n3", + "label": {"content": "DEPARTMENT_NAME", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 139.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 233.0, + "y": -82.72317}], + "height": 89.96875, + "id": "n1", + "label": {"content": "RS-1", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162.0, + "x": 0.0, + "y": 0.0}, + "width": 162.0, + "x": 232.0, + "y": -152.69191}, + {"columns": [{"height": 16.0, + "id": "n2::n0", + "label": {"content": "remote", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 1.0, + "y": -117.9692}], + "height": 41.96875, + "id": "n2", + "label": {"content": "source", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162.0, + "x": 0.0, + "y": 0.0}, + "width": 162.0, + "x": 0.0, + "y": -139.93794}, + {"columns": [{"height": 16.0, + "id": "n3::n0", + "label": {"content": "department_name", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 131.0, + "x": 0.0, + "y": 0.0}, + "width": 160.0, + "x": 1.0, + "y": -189.93825}], + "height": 41.96875, + "id": "n3", + "label": {"content": "department", + "fontFamily": "Segoe ", + "UI ": + "Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162.0, + "x": 0.0, + "y": 0.0}, + "width": 162.0, + "x": 0.0, + "y": -211.907}]}, + "listIdMap": {"n0": ["4"], + "n0::n0": ["5"], + "n0::n1": ["6"], + "n1": ["27"], + "n1::n0": ["29"], + "n1::n1": ["28_0"], + "n1::n2": ["28_1"], + "n1::n3": ["28_2"], + "n2": ["17"], + "n2::n0": ["19"], + "n3": ["8"], + "n3::n0": ["10"]}, + "relationshipIdMap": {"e0": "fdd", + "e1": "fdd", + "e2": "fdd", + "e3": "fdd"}, + "tooltip": {}}, + "mode": "global", + "sqlflow": {"dbobjs": [{"alias": "E", + "columns": [{"coordinates": [{"hashCode": "0", + "x": 7, + "y": 7}, + {"hashCode": "0", + "x": 7, + "y": 22}], + "id": "5", + "name": "DEPARTMENT_ID"}, + {"coordinates": [{"hashCode": "0", + "x": 4, + "y": 9}, + {"hashCode": "0", + "x": 4, + "y": 20}], + "id": "6", + "name": "LAST_NAME"}], + "coordinates": [{"hashCode": "0", + "x": 5, + "y": 7}, + {"hashCode": "0", + "x": 5, + "y": 18}], + "id": "4", + "name": "EMPLOYEES", + "type": "table"}, + {"alias": "D", + "columns": [{"coordinates": [{"hashCode": "0", + "x": 7, + "y": 25}, + {"hashCode": "0", + "x": 7, + "y": 40}], + "id": "9", + "name": "DEPARTMENT_ID"}, + {"coordinates": [{"hashCode": "0", + "x": 4, + "y": 39}, + {"hashCode": "0", + "x": 4, + "y": 56}], + "id": "10", + "name": "DEPARTMENT_NAME"}], + "coordinates": [{"hashCode": "0", + "x": 6, + "y": 18}, + {"hashCode": "0", + "x": 6, + "y": 30}], + "id": "8", + "name": "DEPARTMENT", + "type": "table"}, + {"alias": "S", + "columns": [{"coordinates": [{"hashCode": "0", + "x": 13, + "y": 5}, + {"hashCode": "0", + "x": 13, + "y": 18}], + "id": "18", + "name": "LOCATION_ID"}, + {"coordinates": [{"hashCode": "0", + "x": 10, + "y": 9}, + {"hashCode": "0", + "x": 10, + "y": 17}], + "id": "19", + "name": "REMOTE"}, + {"coordinates": [{"hashCode": "0", + "x": 10, + "y": 18}, + {"hashCode": "0", + "x": 10, + "y": 33}], + "id": "20", + "name": "DEPARTMENT_ID"}], + "coordinates": [{"hashCode": "0", + "x": 11, + "y": 7}, + {"hashCode": "0", + "x": 11, + "y": 15}], + "id": "17", + "name": "SOURCE", + "type": "table"}, + {"alias": "L", + "columns": [{"coordinates": [{"hashCode": "0", + "x": 13, + "y": 21}, + {"hashCode": "0", + "x": 13, + "y": 25}], + "id": "23", + "name": "ID"}], + "coordinates": [{"hashCode": "0", + "x": 12, + "y": 13}, + {"hashCode": "0", + "x": 12, + "y": 23}], + "id": "22", + "name": "LOCATION", + "type": "table"}, + {"columns": [{"coordinates": [{"hashCode": "0", + "x": 2, + "y": 16}, + {"hashCode": "0", + "x": 2, + "y": 31}], + "id": "29", + "name": "REMOTE"}, + {"coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 31}], + "id": "26", + "name": "RELATIONROWS", + "source": "SYSTEM"}, + {"coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 14}], + "id": "28_0", + "name": "LAST_NAME"}, + {"coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 14}], + "id": "28_1", + "name": "DEPARTMENT_ID"}, + {"coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 14}], + "id": "28_2", + "name": "DEPARTMENT_NAME"}], + "coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 31}], + "id": "27", + "name": "RS-1", + "type": "select_list"}], + "dbvendor": "dbvoracle", + "relationships": [{"effectType": "select", + "id": "1", + "sources": [{"column": "DEPARTMENT_NAME", + "coordinates": [{"hashCode": "0", + "x": 4, + "y": 39}, + {"hashCode": "0", + "x": 4, + "y": 56}], + "id": "10", + "parentId": "8", + "parentName": "DEPARTMENT"}], + "target": {"column": "DEPARTMENT_NAME", + "coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 14}], + "id": "28_2", + "parentId": "27", + "parentName": "RS-1"}, + "type": "fdd"}, + {"effectType": "select", + "id": "2", + "sources": [{"column": "REMOTE", + "coordinates": [{"hashCode": "0", + "x": 10, + "y": 9}, + {"hashCode": "0", + "x": 10, + "y": 17}], + "id": "19", + "parentId": "17", + "parentName": "SOURCE"}], + "target": {"column": "REMOTE", + "coordinates": [{"hashCode": "0", + "x": 2, + "y": 16}, + {"hashCode": "0", + "x": 2, + "y": 31}], + "id": "29", + "parentId": "27", + "parentName": "RS-1"}, + "type": "fdd"}, + {"effectType": "join", + "id": "3", + "sources": [{"column": "REMOTE", + "coordinates": [{"hashCode": "0", + "x": 10, + "y": 9}, + {"hashCode": "0", + "x": 10, + "y": 17}], + "id": "19", + "parentId": "17", + "parentName": "SOURCE"}], + "target": {"column": "REMOTE", + "coordinates": [{"hashCode": "0", + "x": 2, + "y": 16}, + {"hashCode": "0", + "x": 2, + "y": 31}], + "id": "29", + "parentId": "27", + "parentName": "RS-1"}, + "type": "fdd"}, + {"effectType": "select", + "id": "4", + "sources": [{"column": "LAST_NAME", + "coordinates": [{"hashCode": "0", + "x": 4, + "y": 9}, + {"hashCode": "0", + "x": 4, + "y": 20}], + "id": "6", + "parentId": "4", + "parentName": "EMPLOYEES"}], + "target": {"column": "LAST_NAME", + "coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 14}], + "id": "28_0", + "parentId": "27", + "parentName": "RS-1"}, + "type": "fdd"}, + {"effectType": "select", + "id": "5", + "sources": [{"column": "DEPARTMENT_ID", + "coordinates": [{"hashCode": "0", + "x": 7, + "y": 7}, + {"hashCode": "0", + "x": 7, + "y": 22}], + "id": "5", + "parentId": "4", + "parentName": "EMPLOYEES"}], + "target": {"column": "DEPARTMENT_ID", + "coordinates": [{"hashCode": "0", + "x": 2, + "y": 8}, + {"hashCode": "0", + "x": 2, + "y": 14}], + "id": "28_1", + "parentId": "27", + "parentName": "RS-1"}, + "type": "fdd"}]}, + "summary": {"column": 8, + "database": 0, + "mostRelationTables": [{"table": "SOURCE"}, + {"table": "EMPLOYEES"}, + {"table": "DEPARTMENT"}], + "process": 0, + "relationship": 4, + "schema": 0, + "table": 4, + "view": 0}}, + "sessionId": "060b90a2beabc1d2f11350e55c19e9b8368dfbd224f3b24a05ead4663f45b362_1653108177816"} \ No newline at end of file diff --git a/demos/top-level-select-list/data-lineage-xml-by-java-dlineage.xml b/demos/top-level-select-list/data-lineage-xml-by-java-dlineage.xml new file mode 100644 index 0000000..62aa3bd --- /dev/null +++ b/demos/top-level-select-list/data-lineage-xml-by-java-dlineage.xml @@ -0,0 +1,46 @@ + + + + + +
+ + + +
+ + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demos/top-level-select-list/lineage_cloud.py b/demos/top-level-select-list/lineage_cloud.py new file mode 100644 index 0000000..678a8c7 --- /dev/null +++ b/demos/top-level-select-list/lineage_cloud.py @@ -0,0 +1,50 @@ +import requests +import json +import pprint + +# Please change the IP to the server where Gudu SQLFlow on-premise version is setup +sqlflow_cloud_server = 'https://api.gudusoft.com' +sqlflow_generate_token = sqlflow_cloud_server + '/gspLive_backend/user/generateToken' +sqlflow_api = sqlflow_cloud_server + '/gspLive_backend/sqlflow/generation/sqlflow/graph' + + +# Please check here for detailed explanation of all parameters: https://github.com/sqlparser/sqlflow_public/tree/master/api/python/basic +userId = 'your user id here' +# This is your screct key: https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md +screctKey = 'your secrect key here' +dbvendor = 'dbvoracle' +ignoreRecordSet = 'True' +ignoreFunction = 'True' +showRelationType = 'fdd' +simpleOutput = 'False' +sqltext = """ +select data.*, location.remote +from ( + select e.last_name, e.department_id, d.department_name + from employees e + left outer join department d + on (e.department_id = d.department_id) +) data inner join +( + select s.remote,s.department_id + from source s + inner join location l + on s.location_id = l.id +) location on data.department_id = location.department_id; +""" + +mapA = {'secretKey': screctKey, 'userId': userId} +header_dict = {"Content-Type": "application/x-www-form-urlencoded; charset=utf8"} +r = requests.post(sqlflow_generate_token, data=mapA, headers=header_dict) +rs = eval(r.text) +# print(rs) +tk = rs.get("token") + +data = {'dbvendor': dbvendor, 'ignoreRecordSet': ignoreRecordSet, 'simpleOutput': simpleOutput, + 'showRelationType': showRelationType, 'token': tk, 'userId': userId, 'sqltext': sqltext} +datastr = json.dumps(data) +response = requests.post(sqlflow_api, data=eval(datastr)) +jsonStr = response.json() + +pp = pprint.PrettyPrinter(width=41, compact=True) +pp.pprint(jsonStr) diff --git a/demos/top-level-select-list/lineage_on_premise.py b/demos/top-level-select-list/lineage_on_premise.py new file mode 100644 index 0000000..6e72218 --- /dev/null +++ b/demos/top-level-select-list/lineage_on_premise.py @@ -0,0 +1,41 @@ +import requests +import json +import pprint + +# Please change the IP to the server where Gudu SQLFlow on-premise version is setup +sqlflow_on_premise_server = 'http://127.0.0.1:8081' +sqlflow_api = sqlflow_on_premise_server + '/gspLive_backend/sqlflow/generation/sqlflow/graph' + +# Please check here for detailed explanation of all parameters: https://github.com/sqlparser/sqlflow_public/tree/master/api/python/basic +userId = 'gudu|0123456789' +dbvendor = 'dbvoracle' +ignoreRecordSet = 'True' +ignoreFunction = 'True' +showRelationType = 'fdd' +simpleOutput = 'False' +sqltext = """ +select data.*, location.remote +from ( + select e.last_name, e.department_id, d.department_name + from employees e + left outer join department d + on (e.department_id = d.department_id) +) data inner join +( + select s.remote,s.department_id + from source s + inner join location l + on s.location_id = l.id +) location on data.department_id = location.department_id; +""" + + +data = {'dbvendor': dbvendor, 'ignoreRecordSet': ignoreRecordSet, 'simpleOutput': simpleOutput, + 'showRelationType': showRelationType, 'userId': userId, 'sqltext': sqltext} +datastr = json.dumps(data) +response = requests.post(sqlflow_api, data=eval(datastr)) +jsonStr = response.json() +# print(jsonStr) + +pp = pprint.PrettyPrinter(width=41, compact=True) +pp.pprint(jsonStr) diff --git a/demos/top-level-select-list/sqlflow-ui-download-json.png b/demos/top-level-select-list/sqlflow-ui-download-json.png new file mode 100644 index 0000000..f2fa11d Binary files /dev/null and b/demos/top-level-select-list/sqlflow-ui-download-json.png differ diff --git a/demos/top-level-select-list/sqlflow-ui-generated.json b/demos/top-level-select-list/sqlflow-ui-generated.json new file mode 100644 index 0000000..5978418 --- /dev/null +++ b/demos/top-level-select-list/sqlflow-ui-generated.json @@ -0,0 +1,832 @@ +{ + "code": 200, + "data": { + "mode": "global", + "summary": { + "schema": 0, + "process": 0, + "database": 0, + "view": 0, + "mostRelationTables": [ + { + "table": "SOURCE" + }, + { + "table": "EMPLOYEES" + }, + { + "table": "DEPARTMENT" + } + ], + "column": 8, + "relationship": 4, + "table": 4 + }, + "sqlflow": { + "dbvendor": "dbvoracle", + "dbobjs": [ + { + "id": "4", + "name": "EMPLOYEES", + "alias": "E", + "type": "table", + "columns": [ + { + "id": "5", + "name": "DEPARTMENT_ID", + "coordinates": [ + { + "x": 6, + "y": 7, + "hashCode": "0" + }, + { + "x": 6, + "y": 22, + "hashCode": "0" + } + ] + }, + { + "id": "6", + "name": "LAST_NAME", + "coordinates": [ + { + "x": 3, + "y": 9, + "hashCode": "0" + }, + { + "x": 3, + "y": 20, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 4, + "y": 7, + "hashCode": "0" + }, + { + "x": 4, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "id": "8", + "name": "DEPARTMENT", + "alias": "D", + "type": "table", + "columns": [ + { + "id": "9", + "name": "DEPARTMENT_ID", + "coordinates": [ + { + "x": 6, + "y": 25, + "hashCode": "0" + }, + { + "x": 6, + "y": 40, + "hashCode": "0" + } + ] + }, + { + "id": "10", + "name": "DEPARTMENT_NAME", + "coordinates": [ + { + "x": 3, + "y": 39, + "hashCode": "0" + }, + { + "x": 3, + "y": 56, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 5, + "y": 18, + "hashCode": "0" + }, + { + "x": 5, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "17", + "name": "SOURCE", + "alias": "S", + "type": "table", + "columns": [ + { + "id": "18", + "name": "LOCATION_ID", + "coordinates": [ + { + "x": 12, + "y": 5, + "hashCode": "0" + }, + { + "x": 12, + "y": 18, + "hashCode": "0" + } + ] + }, + { + "id": "19", + "name": "REMOTE", + "coordinates": [ + { + "x": 9, + "y": 9, + "hashCode": "0" + }, + { + "x": 9, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "20", + "name": "DEPARTMENT_ID", + "coordinates": [ + { + "x": 9, + "y": 18, + "hashCode": "0" + }, + { + "x": 9, + "y": 33, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 10, + "y": 7, + "hashCode": "0" + }, + { + "x": 10, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "22", + "name": "LOCATION", + "alias": "L", + "type": "table", + "columns": [ + { + "id": "23", + "name": "ID", + "coordinates": [ + { + "x": 12, + "y": 21, + "hashCode": "0" + }, + { + "x": 12, + "y": 25, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 11, + "y": 13, + "hashCode": "0" + }, + { + "x": 11, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "27", + "name": "RS-1", + "type": "select_list", + "columns": [ + { + "id": "29", + "name": "REMOTE", + "coordinates": [ + { + "x": 1, + "y": 16, + "hashCode": "0" + }, + { + "x": 1, + "y": 31, + "hashCode": "0" + } + ] + }, + { + "id": "26", + "name": "RELATIONROWS", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 31, + "hashCode": "0" + } + ], + "source": "SYSTEM" + }, + { + "id": "28_0", + "name": "LAST_NAME", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 14, + "hashCode": "0" + } + ] + }, + { + "id": "28_1", + "name": "DEPARTMENT_ID", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 14, + "hashCode": "0" + } + ] + }, + { + "id": "28_2", + "name": "DEPARTMENT_NAME", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 14, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 31, + "hashCode": "0" + } + ] + } + ], + "relationships": [ + { + "id": "501", + "type": "fdd", + "effectType": "select", + "target": { + "id": "28_2", + "column": "DEPARTMENT_NAME", + "parentId": "27", + "parentName": "RS-1", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 14, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "10", + "column": "DEPARTMENT_NAME", + "parentId": "8", + "parentName": "DEPARTMENT", + "coordinates": [ + { + "x": 3, + "y": 39, + "hashCode": "0" + }, + { + "x": 3, + "y": 56, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "502", + "type": "fdd", + "effectType": "select", + "target": { + "id": "29", + "column": "REMOTE", + "parentId": "27", + "parentName": "RS-1", + "coordinates": [ + { + "x": 1, + "y": 16, + "hashCode": "0" + }, + { + "x": 1, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "19", + "column": "REMOTE", + "parentId": "17", + "parentName": "SOURCE", + "coordinates": [ + { + "x": 9, + "y": 9, + "hashCode": "0" + }, + { + "x": 9, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "503", + "type": "fdd", + "effectType": "select", + "target": { + "id": "28_1", + "column": "DEPARTMENT_ID", + "parentId": "27", + "parentName": "RS-1", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 14, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "5", + "column": "DEPARTMENT_ID", + "parentId": "4", + "parentName": "EMPLOYEES", + "coordinates": [ + { + "x": 6, + "y": 7, + "hashCode": "0" + }, + { + "x": 6, + "y": 22, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "504", + "type": "fdd", + "effectType": "select", + "target": { + "id": "28_0", + "column": "LAST_NAME", + "parentId": "27", + "parentName": "RS-1", + "coordinates": [ + { + "x": 1, + "y": 8, + "hashCode": "0" + }, + { + "x": 1, + "y": 14, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "6", + "column": "LAST_NAME", + "parentId": "4", + "parentName": "EMPLOYEES", + "coordinates": [ + { + "x": 3, + "y": 9, + "hashCode": "0" + }, + { + "x": 3, + "y": 20, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "505", + "type": "fdd", + "effectType": "join", + "target": { + "id": "29", + "column": "REMOTE", + "parentId": "27", + "parentName": "RS-1", + "coordinates": [ + { + "x": 1, + "y": 16, + "hashCode": "0" + }, + { + "x": 1, + "y": 31, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "19", + "column": "REMOTE", + "parentId": "17", + "parentName": "SOURCE", + "coordinates": [ + { + "x": 9, + "y": 9, + "hashCode": "0" + }, + { + "x": 9, + "y": 17, + "hashCode": "0" + } + ] + } + ] + } + ] + }, + "graph": { + "relationshipIdMap": { + "e0": "fdd", + "e1": "fdd", + "e2": "fdd", + "e3": "fdd" + }, + "elements": { + "tables": [ + { + "columns": [ + { + "height": 16, + "id": "n0::n0", + "label": { + "content": "department_id", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 107, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -46.00015 + }, + { + "height": 16, + "id": "n0::n1", + "label": { + "content": "last_name", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 80, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -30.000149 + } + ], + "height": 57.96875, + "id": "n0", + "label": { + "content": "employees", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -67.9689 + }, + { + "columns": [ + { + "height": 16, + "id": "n1::n0", + "label": { + "content": "remote", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 233, + "y": -130.72316 + }, + { + "height": 16, + "id": "n1::n1", + "label": { + "content": "LAST_NAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 86, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 233, + "y": -114.72317 + }, + { + "height": 16, + "id": "n1::n2", + "label": { + "content": "DEPARTMENT_ID", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 116, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 233, + "y": -98.72317 + }, + { + "height": 16, + "id": "n1::n3", + "label": { + "content": "DEPARTMENT_NAME", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 139, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 233, + "y": -82.72317 + } + ], + "height": 89.96875, + "id": "n1", + "label": { + "content": "RS-1", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 232, + "y": -152.69191 + }, + { + "columns": [ + { + "height": 16, + "id": "n2::n0", + "label": { + "content": "remote", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 61, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -117.9692 + } + ], + "height": 41.96875, + "id": "n2", + "label": { + "content": "source", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -139.93794 + }, + { + "columns": [ + { + "height": 16, + "id": "n3::n0", + "label": { + "content": "department_name", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 13.96875, + "width": 131, + "x": 0, + "y": 0 + }, + "width": 160, + "x": 1, + "y": -189.93825 + } + ], + "height": 41.96875, + "id": "n3", + "label": { + "content": "department", + "fontFamily": "Segoe UI Symbol", + "fontSize": "12", + "height": 17.96875, + "width": 162, + "x": 0, + "y": 0 + }, + "width": 162, + "x": 0, + "y": -211.907 + } + ], + "edges": [ + { + "id": "e0", + "sourceId": "n2::n0", + "targetId": "n1::n0" + }, + { + "id": "e1", + "sourceId": "n3::n0", + "targetId": "n1::n3" + }, + { + "id": "e2", + "sourceId": "n0::n1", + "targetId": "n1::n1" + }, + { + "id": "e3", + "sourceId": "n0::n0", + "targetId": "n1::n2" + } + ] + }, + "tooltip": { + + }, + "listIdMap": { + "n0": [ + "4" + ], + "n0::n0": [ + "5" + ], + "n0::n1": [ + "6" + ], + "n1": [ + "27" + ], + "n1::n0": [ + "29" + ], + "n1::n1": [ + "28_0" + ], + "n1::n2": [ + "28_1" + ], + "n1::n3": [ + "28_2" + ], + "n2": [ + "17" + ], + "n2::n0": [ + "19" + ], + "n3": [ + "8" + ], + "n3::n0": [ + "10" + ] + } + } + }, + "sessionId": "2d6b60f7774269d22aa54fd50ddbb1e7a7ce645eaf95e0469dd82c10cc91f64c_1653098524225" +} \ No newline at end of file diff --git a/demos/top-level-select-list/sqlflow-ui-paste-sql.png b/demos/top-level-select-list/sqlflow-ui-paste-sql.png new file mode 100644 index 0000000..97710e8 Binary files /dev/null and b/demos/top-level-select-list/sqlflow-ui-paste-sql.png differ diff --git a/demos/top-level-select-list/sqlflow-ui-setting.png b/demos/top-level-select-list/sqlflow-ui-setting.png new file mode 100644 index 0000000..176fab6 Binary files /dev/null and b/demos/top-level-select-list/sqlflow-ui-setting.png differ diff --git a/demos/top-level-select-list/sqlflow-ui-top-level-select-list-data-lineage.png b/demos/top-level-select-list/sqlflow-ui-top-level-select-list-data-lineage.png new file mode 100644 index 0000000..da0e0f3 Binary files /dev/null and b/demos/top-level-select-list/sqlflow-ui-top-level-select-list-data-lineage.png differ diff --git a/doc/3 types of data lineage models.md b/doc/3 types of data lineage models.md new file mode 100644 index 0000000..3c6e2b2 --- /dev/null +++ b/doc/3 types of data lineage models.md @@ -0,0 +1,36 @@ +In order to meet the user's various requirements about data lineage analysis, it is necessary to divide the SQLFlow data lineage model into several levels, each fitting a specific requirement. + +## 1. The complete data lineage model + +In this model, SQLFlow generates the data lineage includes all detailed information such as the RESULT SET generated during a SELECT statement, FUNCTION CALL used to calculate the new column value based on the input column, CASE EXPRESSION used to transform the data from one column to another, and so on. + +This complete lineage model is the base of all other higher level lineage models which only includes some lineages in this complete model by omitting or aggregating some relations and entities in this model. + +The higher level model is not only remove some relations and entities but also merge some relations to create a new relation. The most important entity introduced in the higher level model is PROCESS which is a SQL query/statement that do the transformation. The higher level model use the SQL query as the smallest unit to tells you how the data is transffered from one table/column to the other. On the other hand, the complete model tells you how data is transferred inside a SQL query. + +When analyzing data lineage, the complete model is always generated since all other higher level models are based on this model. However, the complete model is not suitable to present to the user in a diagram if it includes too many entities and relations. The reason is: + +1. Diagram includes thousands of entities and relations is a chaos and almost impossible to navigate in a single picture. +2. It's time comsuing and maybe impossible for the SQLFlow to layout the complete model with thousands of realtions. + +The complete model is good when analyzing the SQL query or stored procedure less than 1000 thousands code lines. In this model, you can see all detailed information you need. For project includes thousands of stored procedures, It is much better to use the higher level model to visualize the dataflow for a specific table/column. + +## 2. The column-level lineage model + +As it name implied, this model traces the dataflow from column to column based on the SQL statement. In other words, from this model, you can see what SQL statement is used to move/impact data from one column to the other. + +This model only includes 3 kinds of entity: the source column, the target column and the SQL statement( we call it PROCESS in the model) and the relation among them. + +If you want to see how data is moved/impacted inside the SQL statement, you can use the complete model of this SQL statement to find more. + +## 3. The table-level lineage model + +As it name implied, this model traces the dataflow from table totable based on the SQL statement. In other words, from this model, you can see what SQL statement is used to move/impact data from onetable to the other. + +This model only includes 3 kinds of entity: the source table, the target table and the SQL statement( we call it PROCESS in the model) and the relation among them. + +If you want to see how data is moved inside the SQL statement, you can use the complete model of this SQL statement to find more. + +## 4. SQLFlow UI + +![image.png](https://images.gitee.com/uploads/images/2021/0707/145133_6f1ed32d_8136809.png) diff --git a/doc/Data lineage model reference.md b/doc/Data lineage model reference.md new file mode 100644 index 0000000..34353ab --- /dev/null +++ b/doc/Data lineage model reference.md @@ -0,0 +1,5 @@ +Version 1.3.5 - 2021/12/05 + +Copyright 2012 - 2021 | [Gudu software](https://www.gudusoft.com) | All Rights Reserved + +https://www.gudusoft.com diff --git a/doc/a-column-level-data-lineage-model.docx b/doc/a-column-level-data-lineage-model.docx new file mode 100644 index 0000000..a83dc0f Binary files /dev/null and b/doc/a-column-level-data-lineage-model.docx differ diff --git a/doc/basic-concepts/1-introduction.md b/doc/basic-concepts/1-introduction.md new file mode 100644 index 0000000..8f31eda --- /dev/null +++ b/doc/basic-concepts/1-introduction.md @@ -0,0 +1,161 @@ +## Introduction + +SQLFlow generates data lineage by analyzing SQL queries and stored procedures. + +The entity in the data lineage model includes table, column, function, relation and other entities . The combination of the entity and dataflow shows the lineage from one table/column to another. + +![image.png](https://images.gitee.com/uploads/images/2021/0706/171437_139f041e_8136809.png) + +[toc] + +## 1. A dataflow unit + +A dataflow unit includes a source entity, a target entity and a dataflow type between them. + +```sql +SELECT p.FirstName +from Person.Person AS p +``` + +This is the dataflow generated for the above SQL query. + +``` +person.persion.FirstName -> direct -> RS-1.FirstName +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1204/185152_c18593ce_8136809.png) + +### 1.1 Source, target entity + +Source and target entity usually referes to table, view and other relations such as Common Table Expression (CTE), result set generated during the execution of the query. It may also refers to a file in the HDFS system and etc. + +### 1.2 Relationship types + +There are several relationship types in SQLFlow, the dataflow type is the most important realtionship type which is used to represents the dataflow between 2 objects, and there are two dataflow types: direct and indirect. + +SQLFlow also supports other relationship types, such as join, function call and so on. + +#### 1.2.1 Direct dataflow + +The direct dataflow means the data of the target entity comes directly from the source entity. + +In the above diagram, the data of `RS-1.FirstName` comes from the `Person.FirstName` directly. + +An arrow is used to represent a direct dataflow in the diagram: + +![image.png](https://images.gitee.com/uploads/images/2021/1204/202053_bfe8900f_8136809.png) + +#### 1.2.2 Indirect dataflow + +The indirect dataflow means the data of the target column is not comes from the source column, but the data of the source column/table impact the result data of the target column. + +A dotted line arrow is used to represent an indirect dataflow in the diagram: + +![image.png](https://images.gitee.com/uploads/images/2021/1204/202348_3a9d1e71_8136809.png) + +The source column in the indirect dataflow usually appears in the following clause: + +- Where clause +- Group by clause +- Winddows function +- Join condition + +```sql +SELECT deptno, COUNT() num_emp, SUM(SAL) sal_sum +FROM scott.emp +GROUP BY deptno +``` + +The value of COUNT() and SUM(SAL) is impacted by the value of column `deptno` in the group by clause. So the indirect dataflows will be created like this: + +``` +scott.emp.deptno -> indirect -> COUNT() +scott.emp.deptno -> indirect -> SUM(SAL) +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/174012_ba0c83f4_8136809.png) + +For other indirect dataflow types, we will discuss later. + +#### 1.2.3 Join + +The `join` relationship build a link between 2 or more columns in the join condition. Striclty speaking, the relation is not a dataflow type. + +```sql + select b.teur + from tbl a left join TT b on (a.key=b.key) +``` + +A join relationship will be created after analzying the above SQL. It indicates a join relationship betwee `tbl.key` and `TT.key`. + +![image.png](https://images.gitee.com/uploads/images/2021/0711/185405_036c2a1a_8136809.png) + +#### 1.2.4 function call + +## 2. The entity in dataflow + +When build dataflow between 2 entities: the source and target entity. They can be column to column, or, table to colum, or table to table. + +### 2.1 column to column + +This is the most often cases. Both entites in a dataflow are columns. + +### 2.2 table to column + +When we say a table impact the value of a column, we usually means the total number of rows of a table impact the value of a column, usually, this column is derived from a COUNT() function. + +```sql +SELECT COUNT() num_emp +FROM scott.emp +``` + +A table to column dataflow is represented by using a RelationRows pseduo column. This build an indirect dataflow from scott.emp.RelationRows to RS-1.num_emp + +```sql +scott.emp.RelationRows -> indirect -> COUNT() -> RS-1.num_emp +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/174427_2f800ff4_8136809.png) + +### 2.3 table to table + +Sometimes, there will be a dataflow between 2 tables. For example, in an `alter table rename` SQL statement, a table to table dataflow will be created. Acutally, a table to table dataflow is represented by a column to column dataflow using the `RelationRows` pseudo column. + +```sql +alter table t2 rename to t3; +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1204/231703_d06e3c39_8136809.png) + +## 3. Data lineage + +A data lineage consists of lots of basic dataflow units. + +```sql +CREATE VIEW vsal +AS + SELECT a.deptno "Department", + a.num_emp / b.total_count "Employees", + a.sal_sum / b.total_sal "Salary" + FROM (SELECT deptno, + Count() num_emp, + SUM(sal) sal_sum + FROM scott.emp + WHERE city = 'NYC' + GROUP BY deptno) a, + (SELECT Count() total_count, + SUM(sal) total_sal + FROM scott.emp + WHERE city = 'NYC') b +``` + +The data lineage diagram: + +![image.png](https://images.gitee.com/uploads/images/2021/0711/221337_e8f731a5_8136809.png) + +The output also available in XML or JSON format . + +## 4. References + +1. xml code used in this article is generated by [DataFlowAnalyzer](https://github.com/sqlparser/gsp_demo_java/tree/master/src/main/java/demos/dlineage) tools +2. digram used in this article is generated by the [Gudu SQLFlow Cloud version](https://sqlflow.gudusoft.com/) \ No newline at end of file diff --git a/doc/basic-concepts/10-transforms.md b/doc/basic-concepts/10-transforms.md new file mode 100644 index 0000000..089030d --- /dev/null +++ b/doc/basic-concepts/10-transforms.md @@ -0,0 +1,44 @@ +## Transforms + +[toc] + +### 1. What is transform? + +In the SQLFlow, transform is the code snippet that transform the data from source to target. +```sql +SELECT SUM(Quantity) as total_quantity FROM OrderDetails; +``` + +The code snippet `SUM(Quantity)` is a transform that transform the data from the source `OrderDetails.Quantity` to the target `total_quantity`. + +![transform sample](../../assets/images/get-started-transform1.png) + + +Transform can be as simple as a function or as complex as a sub-query and stored procedure, it shows the data transformation relationship between the source and target, and tells you how the data is transformed. + +### 2. How transform is organized in the data lineage model? + +In the data lineage model, a transform is the sub-node of a relationship node. +More than one transform can be generated for a relationship node. +```xml + + + + + + SUM(Quantity) + + + + +``` + + + + + + + + + + diff --git a/doc/basic-concepts/11-constant-and-variables.md b/doc/basic-concepts/11-constant-and-variables.md new file mode 100644 index 0000000..648031a --- /dev/null +++ b/doc/basic-concepts/11-constant-and-variables.md @@ -0,0 +1,78 @@ +## 1 Constant + +Constants used in the SQL statement will be collected and saved in a pseudo table: `constantTable`. +Each SQL statement will create a `constantTable` table to save the constants used in the SQL statement. + +So SQLFlow able to generate the data flow to trace the constant value. +> Constants only will be collected when the /showConstant is set to true in the SQLFlow. +and constants used in the insert statement WILL NOT BE collected in order to avoid too many constants even if the /showConstant is set to true. + +>By default, the /showConstant is set to false in the SQLFlow which means constants will not be collected. + +```sql +SELECT 'constant' as field1, 2 as field2; +``` + +a `table` XML tag and type attribute value `constantTable`: + +```xml + + + +
+``` + +```sql +UPDATE table1 t1 JOIN table2 t2 ON t1.field1 = t2.field1 +SET t1.field2='constant' and t1.field3=2; +``` + +a `table` XML tag and type attribute value `constantTable`: + +```xml + + + +
+``` + +## 2 Variables + +SQLFlow able to generate the data flow to trace the variable value which usually used in the stored procedure and return value. +Without the trace of the variable, the data flow is not complete. + +```sql +-- sql server sample query +CREATE FUNCTION dbo.ufnGetInventoryStock(@ProductID int) +RETURNS int +AS +-- Returns the stock level for the product. +BEGIN + DECLARE @ret int; + SELECT @ret = SUM(p.Quantity) + FROM Production.ProductInventory p + WHERE p.ProductID = @ProductID + AND p.LocationID = '6'; + IF (@ret IS NULL) + SET @ret = 0; + RETURN @ret; +END; + +create view v_product as +SELECT ProductModelID, Name, dbo.ufnGetInventoryStock(ProductID)AS CurrentSupply +FROM Production.Product +WHERE ProductModelID BETWEEN 75 and 80; +``` + +As you can see, with the trace of the variable, the data flow from `Production.ProductInventory.Quantity` to the `v_product.CurrentSupply` is complete. + +![trace variable](../../assets/images/get-started-11-variables1.png) + +The data lineage in xml format is as follows: + +```xml + + + +``` + diff --git a/doc/basic-concepts/12-er-diagram.md b/doc/basic-concepts/12-er-diagram.md new file mode 100644 index 0000000..8668dbc --- /dev/null +++ b/doc/basic-concepts/12-er-diagram.md @@ -0,0 +1,34 @@ +```sql +-- oracle sample query +-- Create the Departments table +CREATE TABLE Departments ( +    department_id NUMBER PRIMARY KEY, +    department_name VARCHAR2(50) NOT NULL, +    location VARCHAR2(100) +); + +-- Create the Employees table with a foreign key referencing Departments +CREATE TABLE Employees ( +    employee_id NUMBER PRIMARY KEY, +    first_name VARCHAR2(50) NOT NULL, +    last_name VARCHAR2(50) NOT NULL, +    email VARCHAR2(100) UNIQUE, +    hire_date DATE, +    salary NUMBER(10,2), +    department_id NUMBER, +    CONSTRAINT fk_department +        FOREIGN KEY (department_id) +        REFERENCES Departments(department_id) +); +``` + +A relationship with type attribute value `er` will be generated in the ER diagram. + +```xml + + + + +``` + + diff --git a/doc/basic-concepts/2-direct-dataflow.md b/doc/basic-concepts/2-direct-dataflow.md new file mode 100644 index 0000000..060d267 --- /dev/null +++ b/doc/basic-concepts/2-direct-dataflow.md @@ -0,0 +1,97 @@ +## Direct dataflow + +This article introduces some SQL elements that will generate direct dataflow. + +### 1. Select + +```sql +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` + +the data of target column `"eName"` comes from `scott.emp.empName` , so we have a direct dataflow like this: + +``` +scott.emp.empName -> direct -> RS-1."eName" +``` + +the resultset RS-1 generated by the select list is a relation, which includes columns and rows. + +#### dataflow in XML + +```xml + + + + +
+ + + + + + + + +``` + +The `relation` represents a dataflow from source column with `id=3` to the target column with `id=6` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0702/165128_bef55fd4_8136809.png) + +### 2. Function + +During the dataflow analyzing, `function` plays a key role. It accepts columns as arguments and generate result which maybe a scalar value or a set value. + +```sql +select round(salary) as sal from scott.emp +``` + +A direct dataflow is generated from column salary to the `round` function in the above SQL : + +``` +scott.emp.salary -> direct -> round(salary) -> direct -> sal +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/174849_3c374514_8136809.png) + +#### dataflow in xml + +```xml + + + + +
+ + + + + + + + + + + + + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0702/172358_bc2c88ad_8136809.png) + +if you turn off the `show function` setting with `/if` option, the result is: + +![image.png](https://images.gitee.com/uploads/images/2021/0702/172626_96c936a1_8136809.png) + +### 3. References + +1. xml code used in this article is generated by [DataFlowAnalyzer](https://github.com/sqlparser/gsp_demo_java/tree/master/src/main/java/demos/dlineage) tools +2. digram used in this article is generated by the [Gudu SQLFlow Cloud version](https://sqlflow.gudusoft.com/) \ No newline at end of file diff --git a/doc/basic-concepts/3-indirect-dataflow-and-pseudo-column.md b/doc/basic-concepts/3-indirect-dataflow-and-pseudo-column.md new file mode 100644 index 0000000..effed11 --- /dev/null +++ b/doc/basic-concepts/3-indirect-dataflow-and-pseudo-column.md @@ -0,0 +1,94 @@ +## Indirect dataflow and RelationRows (pseudo column) + +This article introduces some SQL elements that generate indirect dataflow. Indirect dataflow usually is generated from columns used in the where clause, group by clause, aggregate function and etc. + +In order to create indirect dataflow between columns, we introduce a pseudo column: **RelationRows**. + +RelationRows is a pseudo column of a relation used to represents the number of rows in a relation. As it's name indicates, RelationRows is not a real column in the relation(table/resultset and etc). Usually, It is used to represent a dataflow between a column and a relation. + +RelationRows pseudo column can be used in both the source and target relation. + +## 1 RelationsRows in target relation + +Take this SQL for example: + +```sql +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` + +The total number of rows of the select list is impacted by the value of column `sal` in the where clause. So, an indirect dataflow is created like this: + +``` +scott.emp.sal -> indirect -> RS-1.RelationRows +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/120228_c087c542_8136809.png) + +## 2 RelationRows in source relation + +Here is another sample: + +```sql +SELECT count() totalNum, sum(sal) totalSal +FROM scott.emp +``` + +The value of `count()` function and `sum(sal)` function is impacted by the number of rows in the `scott.emp` source table. + +``` +scott.emp.RelationRows -> indirect -> count() +scott.emp.RelationRows -> indirect -> sum(sal) +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/120353_cfebf6b1_8136809.png) + +## 3. Table level dataflow using RelationRows + +RelationRows is also used to represent table level dataflow. + +```sql +alter table t2 rename to t3; +``` + +A table level dataflow is not built on a table, but on the pseudo column `RelationRows` like this: + +```sql +t2.RelationRows -> direct -> t3.RelationRows +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/120446_f7e66732_8136809.png) + +Build a table to table dataflow that using the RelationRows pseudo column for 2 reasons: + +* This pseudo column that used to represent a table to column dataflow can be re-used in a table to table dataflow later when SQLFlow generates a table-level relationship. +* If other columns in the same table are used in a column to column dataflow while this table itself is also in a table to table dataflow, then, this pseudo column will make it possible for a single table to includes both the column to column dataflow and table to table dataflow. + +take this SQL for example + +```sql +create view v1 as select f1 from t2; +alter table t2 rename to t3; +``` + +The first create view statement will generate a column-level dataflow between the table `t2` and view `v1`, + +```sql +t2.f1 -> direct -> RS-1.f1 -> direct -> v1.f1 +``` + +while the second alter table statement will genereate a table-level dataflow between the table t2 and t3. + +```sql +t2.RelationRows -> direct -> t3.RelationRows +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/140840_6f229397_8136809.png) + +As you can see, Table `t2` involved in the column to column dataflow generated by the `create view` statement, It also involved in a table to table dataflow generated by the `alter table` statement. A single table `t2` in the above digram shows that it includes both the column to column dataflow and a table to table dataflow. + +## 4. References + +1. xml code used in this article is generated by [DataFlowAnalyzer](https://github.com/sqlparser/gsp_demo_java/tree/master/src/main/java/demos/dlineage) tools +2. digram used in this article is generated by the [Gudu SQLFlow Cloud version](https://sqlflow.gudusoft.com/) \ No newline at end of file diff --git a/doc/basic-concepts/4-indirect-dataflow-where-group-by.md b/doc/basic-concepts/4-indirect-dataflow-where-group-by.md new file mode 100644 index 0000000..380e47d --- /dev/null +++ b/doc/basic-concepts/4-indirect-dataflow-where-group-by.md @@ -0,0 +1,115 @@ +## Indirect dataflow between columns in where clause and aggregate functions + +### 1. Columns in where clause + +Some columns in the source table in a WHERE clause do not influence the data of the target columns but are crucial for the selected row set numbers, so they should be saved for impact analyses, with an indirect dataflow to the target tables. + +Take this SQL for example: + +```sql +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` + +The total number of rows of the select list is impacted by the value of column `sal` in the where clause. We build an indirect dataflow for this relationship. + +``` +scott.emp.sal -> indirect -> RS-1.RelationRows +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/120228_c087c542_8136809.png) + +### 2. COUNT() + +COUNT() function is an aggregate function that used to calculate the total number of rows of a relation. + +#### 2.1 where clause without group by clause + +```sql +SELECT COUNT() total_num +FROM scott.emp +where city=1 +``` + +In above SQL, two indirect dataflows will be created, +- The value of COUNT() is impacted by the city column in where clause +```sql +scott.emp.city -> indirect -> COUNT() +``` + +- The value of COUNT() is also impacted by the total number of rows of scott.emp table. +```sql +scott.emp.RelationRow -> indirect -> COUNT() +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/150203_a4bbf172_8136809.png) + +#### 2.2 where clause and group by clause + +```sql +SELECT deptno, count() total_num +FROM scott.emp +where city=1 +group by deptno +``` + +As you can see, besides the two indirect dataflows created in the previous SQL, a third indirect dataflow is created using the deptno in the group by clause. + +```sql +scott.emp.city -> indirect -> COUNT() +scott.emp.Relations -> indirect -> COUNT() +scott.emp.deptno -> indirect -> COUNT() +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/150427_bad8e1d6_8136809.png) + +### 3. Other aggregate function + +When creating indirect dataflow, other aggregate functions such as SUM() works **a little difference** to the COUNT() function. + +#### 3.1 where clause and group by clause + +```sql +SELECT deptno, SUM(SAL) sal_sum +FROM scott.emp +where city=1 +group by deptno +``` + +aggregate function such as SUM() calculates the value from a record set determined by the columns used in the group by clause, so deptno column in the group by clause is used to create an indirect dataflow to SUM() function. + +an indirect dataflow is created from deptno to SUM(). + +```sql +scott.emp.deptno -> indirect -> SUM() +``` + +**RelationRows pseudo column will not be used to create an indirect dataflow if group by clause if presented. This is because the value of SUM() function is calculated from a record set determined by the columns used in the group by clause, so the total number of rows of the record set is not needed for the calculation of SUM() function.** + +![image.png](https://images.gitee.com/uploads/images/2021/1210/170231_fd2cfc92_8136809.png) + +#### 3.2 where clause without group by clause + +```sql +SELECT SUM(SAL) sal_sum +FROM scott.emp +where city=1 +``` + +The above SQL means that the whole record set of the table will be used to calculate the value of SUM() function. + +So, two indirect dataflows will be created as below: + +```sql +scott.emp.city -> indirect -> SUM() +scott.emp.RelationRows -> indirect -> SUM() +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/143844_5a1e3bad_8136809.png) + +### 4. Summary + +- Columns in where clause always create an indirect dataflow to all aggregate functions used in the select list. +- RelationRows pseudo column always create an indirect dataflow to COUNT() function, but only create an indirect dataflow to other aggregate functions such as SUM() when the group by clause is not used. (**Maybe COUNT() should be treated the same as other aggregate functions?**) +- Columns in the group by clause always create an indirect dataflow to all aggregate functions used in the select list. \ No newline at end of file diff --git a/doc/basic-concepts/5-dataflow-column-used-in-aggregate-function.md b/doc/basic-concepts/5-dataflow-column-used-in-aggregate-function.md new file mode 100644 index 0000000..4bfc1a9 --- /dev/null +++ b/doc/basic-concepts/5-dataflow-column-used-in-aggregate-function.md @@ -0,0 +1,59 @@ +## Dataflow between column used as aggregate function argument and the aggregate function + +Aggregate function usually takes column as an argument, in this article, we will discuss what's kind of dataflow will be created between the column used as function argument and the aggregate function. + +## 1. COUNT() + +COUNT() may takes a star column, or any column name or even empty argument. + +If the argument is empty or a star column, no dataflow will be generated between the argument and function. +If a column is used as the argument, a direct dataflow will be generated between the column and function +by setting `/treatArgumentsInCountFunctionAsDirectDataflow` option to `true`. + +### 1.1 A direct dataflow + +```sql +SELECT count(empId) total_num FROM scott.emp +``` + +In [SQLFlow Cloud](https://sqlflow.gudusoft.com), a direct dataflow will be generated between the empId column and COUNT() function by default. + +However, in [Dlineage command line tool](https://github.com/sqlparser/gsp_demo_java/releases), a direct dataflow will not be generated between the empId column and COUNT() function by default. + +To enable the direct dataflow in Dlineage command line tool, you can use the `/treatArgumentsInCountFunctionAsDirectDataflow` option. + +``` +scott.emp.empId -> direct -> COUNT() +``` + +This dataflow may seems strange since the result value of COUNT() doesn't depends on the value of empId column. But, this is an option if our users prefer to have such a dataflow. + +![image.png](https://images.gitee.com/uploads/images/2021/1206/225504_c49c3750_8136809.png) + +### 1.2 No dataflow + +You can use an option to decide not to generate a dataflow between empId and COUNT() if preferred. + +Please note that, no matter a direct dataflow is generated between the empId and COUNT() or not. The following indirect dataflow will always be created. + +``` +scott.emp.RelationRows -> indirect -> COUNT() +``` + +## 2. Aggregate function exclude COUNT() + +COUNT() function is a little bit difference when creating dataflow. All other aggregate functions such as SUM() will create a direct dataflow with the column used as the argument. + +```sql +SELECT deptno, SUM(SAL) sal_sum +FROM scott.emp +group by deptno +``` + +A direct dataflow will be created from SAL to SUM(). + +```sql +scott.emp.SAL -> direct -> SUM() +``` + +![image.png](https://images.gitee.com/uploads/images/2021/1206/160142_54580f3a_8136809.png) \ No newline at end of file diff --git a/doc/basic-concepts/6-dataflow-chain.md b/doc/basic-concepts/6-dataflow-chain.md new file mode 100644 index 0000000..f272af9 --- /dev/null +++ b/doc/basic-concepts/6-dataflow-chain.md @@ -0,0 +1,40 @@ +## Dataflow chain + +If the resultset of a subquery or CTE is used in the from clause of the upper-level statement, then the impact of the lower level resultset will be transferred to the upper-level. + +```sql +WITH + cteReports (EmpID, FirstName, LastName, MgrID, EmpLevel) + AS + ( + SELECT EmployeeID, FirstName, LastName, ManagerID, EmpLevel -- resultset1 + FROM Employees + WHERE ManagerID IS NULL + ) +SELECT + count(EmpID), sum(EmpLevel) -- resultset2 +FROM cteReports +``` + +In the CTE, there is an impact relation: + +``` +Employees.ManagerID -> indirect -> RS-1.RelationRows +``` + +Since `cteReports` is used in the from clause of the upper-level statement, then the impact will carry on like this: + +``` +Employees.ManagerID -> indirect -> RS-1.RelationRows -> indirect -> CTE-CTEREPORTS.RelataionRows +``` + +If we ignore the intermediate resultset, the end to end dataflow is : + +``` +Employees.ManagerID -> indirect -> RS-2.COUNT(EmpID) +Employees.ManagerID -> indirect -> RS-2.SUM(EmpLevel) +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/1206/180916_623e6213_8136809.png) \ No newline at end of file diff --git a/doc/basic-concepts/7-intermediate-resultset.md b/doc/basic-concepts/7-intermediate-resultset.md new file mode 100644 index 0000000..5ea560a --- /dev/null +++ b/doc/basic-concepts/7-intermediate-resultset.md @@ -0,0 +1,292 @@ +## Intermediate result set + +[toc] + +### 1. What is intermediate result set? + +In the SQLFlow, intermediate result set is the result set of a select statement. + +The intermediate result set is used to show the details of the data flow and let you know exactly what is going on. + +The intermediate result set is always built in order to create a complete data flow graph. However, you may choose to hide the intermediate result set in the UI in order to make the data flow graph cleaner in a big data flow scenario. + +For example: + +```sql +CREATE VIEW v1 AS +SELECT EmployeeID, FirstName, LastName, ManagerID, EmpLevel -- resultset1 +FROM Employees +WHERE ManagerID IS NULL +``` + +The intermediate result set is: + +``` +EmployeeID, FirstName, LastName, ManagerID, EmpLevel +``` +in the select list and shown in the data flow graph as below: + +![intermediate-resultset](../../assets/images/get-started-intermediate-resultset1.png) + +### CTE example: + +CTE will be treated as a intermediate result set. + +```sql +CREATE VIEW V1 AS +WITH + cteReports (EmpID, FirstName, LastName, MgrID, EmpLevel) + AS + ( + SELECT EmployeeID, FirstName, LastName, ManagerID, EmpLevel -- resultset1 + FROM Employees + WHERE ManagerID IS NULL + ) +SELECT + count(EmpID), sum(EmpLevel) -- resultset2 +FROM cteReports +``` + +So, there are there intermediate result sets: + +1. resultset1: EmployeeID, FirstName, LastName, ManagerID, EmpLevel +2. cte: cteReports (EmpID, FirstName, LastName, MgrID, EmpLevel) +3. resultset2: count(EmpID), sum(EmpLevel) + +And the data flow graph is like this: + +![intermediate-resultset](../../assets/images/get-started-intermediate-resultset2.png) + +### 2. SQL clauses that generate intermediate result set + +The definition of intermediate result set type can be found in the [ResultSetType.java](src/main/java/gudusoft/gsqlparser/dlineage/dataflow/model/ResultSetType.java) + +```java +array, struct, result_of, cte, insert_select, update_select, merge_update, +merge_insert, output, update_set,pivot_table, unpivot_table, alias, rs, function, +case_when; +``` + +#### 1. select list (select_list) +```sql +SELECT EmployeeID, FirstName, LastName, ManagerID, EmpLevel FROM Employees +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `select_list`) + +```xml + + + + + + + +``` + + +#### 2. cte (cte) +[CTE example SQL](#cte-example), the intermediate result set generated: (a `resultset` XML tag and type attribute value `with_cte`) +```xml + + + + + + + + +``` + +#### 3. set clause in update statement (update_set) +```sql +UPDATE table1 t1 JOIN table2 t2 ON t1.field1 = t2.field1 +SET t1.field2=t2.field2 --mysql +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `update-set`) + +```xml + + + + +``` + +![update-set](../../assets/images/get-started-intermediate-resultset3.png) + +#### 4. merge statement (update_select, merge_update) + +```sql +-- bigquery sample SQL +MERGE dataset.DetailedInventory T USING dataset.Inventory S ON T.product = S.product +WHEN NOT MATCHED AND s.quantity < 20 THEN + INSERT(product, quantity, supply_constrained, comments) + VALUES(product, quantity, true, ARRAY>[(DATE('2016-01-01'), 'comment1')]) +WHEN NOT MATCHED THEN + INSERT(product, quantity, supply_constrained) + VALUES(product, quantity, false) +; +``` + +the intermediate result set generated: (a `resultset` XML tag with type attribute value `merge-insert`) + +```xml + + + + + + + + + + + +``` +#### 5. pivot table (pivot_table) + +```sql +-- sql server sample SQL +SELECT * +FROM +( + SELECT salesperson, product, sales_amount + FROM sales +) AS SourceTable +PIVOT +( + SUM(sales_amount) + FOR product IN ([Laptop], [Desktop], [Tablet]) +) AS PivotTable; +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `pivot_table`) + +```xml + + + + + +``` + +#### 6. unpivot table (unpivot_table) + +```sql +-- sql server sample SQL +SELECT ProductID, Quarter, Sales +FROM SalesData +UNPIVOT +( + Sales + FOR Quarter IN (Q1_Sales, Q2_Sales, Q3_Sales, Q4_Sales) +) AS UnpivotedData; +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `unpivot_table`) + +```xml + + + + +``` + +#### 7. table alias (alias) + +The table alias: p (empid_renamed, Q1, Q2, Q3, Q4) in the following SQL statement: +```sql +-- sql server sample SQL +SELECT * +FROM quarterly_sales + PIVOT(SUM(amount) FOR quarter IN ('2023_Q1','2023_Q2','2023_Q3','2023_Q4') + ) AS p (empid_renamed, Q1, Q2, Q3, Q4) +ORDER BY empid_renamed; +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `alias`) + +```xml + + + + + + + +``` + +#### 8. array, struct, result_of, output, rs +Those definition of intermediate result set types are not used in the SQLFlow. + +### 3. resultset output but not a relation + +#### (1) Function +Due to historical design reasons, some SQL clauses will generate a result set but it is not a relation. The most common example is the SQL function that returns a scalar value. + +```sql +SELECT COUNT(*) FROM Employees +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `scalar`) + +```xml + + + +``` + +This result is not a true intermediate result set, but rather a scalar value. However, we can distinguish it from other intermediate result sets by examining the `type` attribute. In this case, the `type` attribute has a value of `function`, indicating that it represents the output of a SQL function rather than a traditional result set. + +This distinction is important for understanding how different SQL operations are represented in the intermediate result structure. While tables and sub-queries typically produce relational result sets, functions often return single values, which are handled differently in the data lineage analysis. + +##### Case expression + +case expression is treated as a special function when considered in the context of data lineage analysis. + +```sql +-- postgres sample SQL +SELECT + employee_id,first_name,last_name,salary, + CASE + WHEN salary < 30000 THEN 'Low' + WHEN salary BETWEEN 30000 AND 60000 THEN 'Medium' + WHEN salary > 60000 THEN 'High' + ELSE 'Unknown' + END AS salary_category +FROM + employees; +``` + +the intermediate result set generated: (a `resultset` XML tag and type attribute value `case_when`) + +```xml + + + +``` + +### 4. List of all intermediate result sets + +A complete list of all intermediate result sets that can be controlled to remove from the data flow graph is as follows: + + +1. XML tag: ``, type attribute value:`select_list` +2. XML tag: ``, type attribute value:`with_cte` +3. XML tag: ``, type attribute value:`update-set` +4. XML tag: ``, type attribute value:`merge-insert` +5. XML tag: ``, type attribute value:`pivot_table` +6. XML tag: ``, type attribute value:`unpivot_table` +7. XML tag: ``, type attribute value:`alias` +8. XML tag: ``, type attribute value:`function` +9. XML tag: ``, type attribute value:`constantTable` +10. XML tag: ``, type attribute value:`variable` + +所有默认生成的 `` and `` 的内容都可以通过设置参数来移除， +所有默认没有生成的例如 `
` with type attribute value `constantTable` 可以通过设置参数来生成。 + + +在 dlineage demo tool 中加入可以控制以上所有参数的设置。形如：/removeResultSetXXX, /includeTableConstantTable, /removeVariable 等。 + +目前在 dlineage demo tool 中的 /s , /i, /if 等参数都是这些参数的一种组合。 \ No newline at end of file diff --git a/doc/basic-concepts/8-join-relation.md b/doc/basic-concepts/8-join-relation.md new file mode 100644 index 0000000..b6dbda7 --- /dev/null +++ b/doc/basic-concepts/8-join-relation.md @@ -0,0 +1,2 @@ +# TODO + diff --git a/doc/basic-concepts/9-temporary-table.md b/doc/basic-concepts/9-temporary-table.md new file mode 100644 index 0000000..ef083b4 --- /dev/null +++ b/doc/basic-concepts/9-temporary-table.md @@ -0,0 +1,25 @@ +## Temporary table + +This section is about how to handle the temporary table in the data lineage analysis. + +Some user like to see the temporary table in the data lineage analysis, but some user don't like to see the temporary table in the data lineage analysis. + +thus we provide an option `/withTemporaryTable` to let user decide whether to output the temporary table in the data lineage analysis. + +Temporary table such as `#temp_table` in SQL Server will not be output in simple output by default. + +If you want to output the temporary table, you can use `/withTemporaryTable` option. + +``` +/withTemporaryTable +``` + +### Temporary table in different database + +Different database products have different syntax for temporary table. + +For example: + +- In SQL Server, the temporary table is like `#temp_table`. +- In MySQL, The temporary table is like `tmp_table`. + diff --git a/doc/basic-concepts/more-dataflow-samples/Handle the dataflow chain.md b/doc/basic-concepts/more-dataflow-samples/Handle the dataflow chain.md new file mode 100644 index 0000000..55b9551 --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/Handle the dataflow chain.md @@ -0,0 +1,21 @@ +### Handle the dataflow chain + +Every relation in the SQL is picked up by the tool, and connected together to show the whole dataflow chain. +Sometimes, we only need to see the end to end relation and ignore all the intermediate relations. + +If we need to convert a fully chained dataflow to an `end to end` dataflow, we may consider the following rules: + +1. A single dataflow chain with the mixed relation types: fdd and fdr. + + ``` + A -> fdd -> B -> fdr -> C -> fdd -> D + ``` + the rule is: if any `fdr` relation appears in the chain, the relation from `A -> D` will be consider as type of `fdr`, otherwise, the final relation is `fdd` for the end to end relation of `A -> D`. +2. If there are multiple chains from `A -> D` + + ``` + A -> fdd -> B1 -> fdr -> C1 -> fdd -> D + A -> fdd -> B2 -> fdr -> C1 -> fdd -> D + A -> fdd -> B3 -> fdd -> C3 -> fdd -> D + ``` + The final relation should choose the `fdd` chain if any. diff --git a/doc/basic-concepts/more-dataflow-samples/case expression (direct).md b/doc/basic-concepts/more-dataflow-samples/case expression (direct).md new file mode 100644 index 0000000..c3f562f --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/case expression (direct).md @@ -0,0 +1,24 @@ +### case expression + +```sql + select + case when a.kamut=1 and b.teur IS null + then 'no locks' + when a.kamut=1 + then b.teur + else 'locks' + end teur + from tbl a left join TT b on (a.key=b.key) +``` + +During the analyzing of dataflow, case expression is treated as a function. The column used inside the case expression will be treated like the arguments of a function. +So for the above SQL, the following relation is discovered: + +``` +tbl.kamut -> direct -> teur +TT.teur -> direct -> teur +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0704/140751_da11dcf1_8136809.png) diff --git a/doc/basic-concepts/more-dataflow-samples/create external table (path).md b/doc/basic-concepts/more-dataflow-samples/create external table (path).md new file mode 100644 index 0000000..b14f370 --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/create external table (path).md @@ -0,0 +1,185 @@ +create table external table usually will use `path` object. + +### snowflake create external + +```sql +create or replace stage exttable_part_stage + url='s3://load/encrypted_files/' + credentials=(aws_key_id='1a2b3c' aws_secret_key='4x5y6z') + encryption=(type='AWS_SSE_KMS' kms_key_id = 'aws/key'); + +create external table exttable_part( + date_part date as to_date(split_part(metadata$filename, '/', 3) + || '/' || split_part(metadata$filename, '/', 4) + || '/' || split_part(metadata$filename, '/', 5), 'YYYY/MM/DD'), + timestamp bigint as (value:timestamp::bigint), + col2 varchar as (value:col2::varchar)) + partition by (date_part) + location=@exttable_part_stage/logs/ + auto_refresh = true + file_format = (type = parquet); +``` + +The data of the external table `exttable_part` comes from the `path ('s3://load/encrypted_files/')` via the stage: `exttable_part_stage` + +``` +path('s3://load/encrypted_files/') -> fdd -> exttable_part_stage (url) -> fdd -> exttable_part(date_part,timestamp,col2) +``` + +#### dataflow in xml + +```xml + + + + + + + + + + +
+ + + +
+ + + + + + + + + + + + + + + + + +``` + +#### diagam + +![image.png](https://images.gitee.com/uploads/images/2021/0709/113955_8d0a8d6d_8136809.png) + +#### table-level lineage + +this SQL is able to create a table-level lineage like this: + +``` +path('s3://load/encrypted_files/') -> process(create stage) -> exttable_part_stage (url) -> process(create external table) -> exttable_part +``` + +![image.png](https://images.gitee.com/uploads/images/2021/0709/114036_31ac2c6c_8136809.png) + +### bigquery create external table + +```sql +CREATE EXTERNAL TABLE dataset.CsvTable OPTIONS ( + format = 'CSV', + uris = ['gs://bucket/path1.csv', 'gs://bucket/path2.csv'] +); +``` + +The data of the external table `dataset.CsvTable` comes from the csv file: `gs://bucket/path1.csv, gs://bucket/path2.csv` + +``` +path (uri='gs://bucket/path1.csv') -> fdd -> dataset.CsvTable +path (uri='gs://bucket/path2.csv') -> fdd -> dataset.CsvTable +``` + +#### dataflow in xml + +```xml + + + + + + + + + + + + + +
+ + + + + + + + + +``` + +#### diagam + +![image.png](https://images.gitee.com/uploads/images/2021/0709/114123_8e3102c6_8136809.png) + +#### table-level lineage + +This SQL is able to create a table-level lineage like this: + +``` +path (uri='gs://bucket/path1.csv') -> query process(create external table) -> dataset.CsvTable +path (uri='gs://bucket/path2.csv') -> query process(create external table) -> dataset.CsvTable +``` + +![image.png](https://images.gitee.com/uploads/images/2021/0709/114151_76369339_8136809.png) + +### Hive load data + +```sql +LOAD DATA LOCAL INPATH /tmp/pv_2008-06-08_us.txt INTO TABLE page_view PARTITION(date='2008-06-08', country='US') +``` + +The data flow is: + +``` +path (uri='/tmp/pv_2008-06-08_us.txt') -> fdd -> page_view(date,country) +``` + +#### dataflow in xml + +```xml + + + + + + + + + +
+ + + + + + + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0709/114357_f3be8f9f_8136809.png) + +#### table-level lineage + +``` +path (uri='/tmp/pv_2008-06-08_us.txt') -> query process(load data) -> page_view +``` + +![image.png](https://images.gitee.com/uploads/images/2021/0709/114422_58785e85_8136809.png) diff --git a/doc/basic-concepts/more-dataflow-samples/create view.md b/doc/basic-concepts/more-dataflow-samples/create view.md new file mode 100644 index 0000000..76ea5ba --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/create view.md @@ -0,0 +1,36 @@ +```sql +create view vEmp(eName) as +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` + +### fdd + +Data in the column `eName` of the view `vEmp` comes from column `empName` of the table `scott.emp` via the chain like this: + +``` +scott.emp.empName -> fdd -> RS-1."eName" -> vEmp.eName +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0704/145716_35ee907e_8136809.png) + +### fdr + +From this query, you will see how the column `sal` in where clause impact the number of rows in the top level view `vEmp`. + +``` +scott.emp.sal -> fdr -> resultset1.PseudoRows -> fdr -> vEmp.PseudoRows +``` + +So, from an end to end point of view, there will be a `fdr` relation between column `sal` and view `vEmp` like this: + +``` +scott.emp.sal -> fdr -> vEmp.PseudoRows +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0704/145810_66a232d3_8136809.png) diff --git a/doc/basic-concepts/more-dataflow-samples/foreign key.md b/doc/basic-concepts/more-dataflow-samples/foreign key.md new file mode 100644 index 0000000..5026717 --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/foreign key.md @@ -0,0 +1,51 @@ +The foreign key in create table statement will create a column-level lineage. + +```sql +CREATE TABLE masteTable +( + masterColumn varchar(3) Primary Key, +); + + +CREATE TABLE foreignTable +( + foreignColumn1 varchar(3) NOT NULL , + foreignColumn2 varchar(3) NOT NULL + FOREIGN KEY (foreignColumn1) REFERENCES masteTable(masterColumn), + FOREIGN KEY (foreignColumn2) REFERENCES masteTable(masterColumn) +) +``` + +The data flow is: + +``` +masteTable.masterColumn -> fdd -> foreignTable.foreignColumn1 +masteTable.masterColumn -> fdd -> foreignTable.foreignColumn2 +``` + +### dataflow in xml + +```xml + + + + +
+ + + +
+ + + + + + + + + +``` + +### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0705/150530_b497fb6c_8136809.png) diff --git a/doc/basic-concepts/more-dataflow-samples/insert overwrite (Hive).md b/doc/basic-concepts/more-dataflow-samples/insert overwrite (Hive).md new file mode 100644 index 0000000..383f4cb --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/insert overwrite (Hive).md @@ -0,0 +1,52 @@ +```sql +INSERT OVERWRITE LOCAL DIRECTORY '/tmp/pv_gender_sum' +SELECT pv_gender_sum.* +FROM pv_gender_sum; +``` + +### column-level lineage + +The data flow is: + +``` +pv_gender_sum(*) -> fdd -> path ( uri='/tmp/pv_gender_sum') +``` + +#### dataflow in xml + +```xml + + + + + + + + +
+ + + + + + + + + + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0705/150952_71d31fcd_8136809.png) + +### table-level lineage + +``` +pv_gender_sum -> query process (insert overwrite) -> path ( uri='/tmp/pv_gender_sum') +``` + + + ![image.png](https://images.gitee.com/uploads/images/2021/0709/140623_6cb3727f_8136809.png) diff --git a/doc/basic-concepts/more-dataflow-samples/join condition (indirect).md b/doc/basic-concepts/more-dataflow-samples/join condition (indirect).md new file mode 100644 index 0000000..22efcce --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/join condition (indirect).md @@ -0,0 +1,27 @@ +### frd relation + +```sql + select b.teur + from tbl a left join TT b on (a.key=b.key) +``` + +Columns in the join condition also effect the number of row in the resultset of the select list just like column in the where clause do. + +So, the following relation will be discoverd in the above SQL. + +``` +tbl.key -> fdr -> resultset.PseudoRows +TT.key -> fdr -> resultset.PseudoRows +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0704/145001_bf601741_8136809.png) + +### join relation + +A join relation will be created after analzye the above SQL. It indicates a join relation betwee `tbl.key` and `TT.key`. + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0711/185405_036c2a1a_8136809.png) diff --git a/doc/basic-concepts/more-dataflow-samples/rename and swap table.md b/doc/basic-concepts/more-dataflow-samples/rename and swap table.md new file mode 100644 index 0000000..484f3ac --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/rename and swap table.md @@ -0,0 +1,64 @@ +```sql +create view v1 as select f1 from t2; +alter table t2 rename to t3; +``` + +### column-level lineage mode + +In order to put a table involved in both column-level lineage and table-level lineage into one picture,we use `PseudoRows` column in order to represent this relation. + +``` +t2.PseudoRows -> fdd -> t3.PseudoRows +``` + +#### diagram + +This is the diagram show lineage in column-level mode. + +![image.png](https://images.gitee.com/uploads/images/2021/0704/180540_c910755a_8136809.png) + +### table-level lineage mode + +If we want to show the table in above SQL in a table-level lineage mode, the relation between 2 tables should be represented by another form like this: + +``` +t2 -> query process (create view) -> v1 +t2 -> query process (alter table rename) -> t3 +``` + +#### diagram + +This is the diagram show lineage in table-level mode. + +![image.png](https://images.gitee.com/uploads/images/2021/0707/145605_6c4f4b22_8136809.png) + +#### dataflow in xml + +```xml + + + + + +
+ + + + + + + + + + + + + + + + + + + + +``` diff --git a/doc/basic-concepts/more-dataflow-samples/snowflake create stream.md b/doc/basic-concepts/more-dataflow-samples/snowflake create stream.md new file mode 100644 index 0000000..35d0049 --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/snowflake create stream.md @@ -0,0 +1,55 @@ +### create stream + +```sql +create stream mystream on table mytable; +``` + +data lineage + +```sql +mytable -> fdd -> mysteam +``` + +#### column level lineage + +```plaintext +mytable.PseudoRows -> fdd -> mysteam.PseudoRows +``` + +#### table level lineage + +```plaintext +mytable -> create stream process -> mystream +``` + +### sample SQLs + +```sql +create stream mystream on table mytable before (timestamp => to_timestamp(40*365*86400)); +create stream mystream on table mytable at (timestamp => to_timestamp_tz('02/02/2019 01:02:03', 'mm/dd/yyyy hh24:mi:ss')); +create stream mystream on table mytable at(offset => -60*5); +create stream mystream on table mytable before(statement => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726'); + +-- external table +-- Create an external table that points to the MY_EXT_STAGE stage. +-- The external table is partitioned by the date (in YYYY/MM/DD format) in the file path. +create external table my_ext_table ( + date_part date as to_date(substr(metadata$filename, 1, 10), 'YYYY/MM/DD'), + ts timestamp as (value:time::timestamp), + user_id varchar as (value:userId::varchar), + color varchar as (value:color::varchar) +) partition by (date_part) + location=@my_ext_stage + auto_refresh = false + file_format=(type=json); + +-- Create a stream on the external table +create stream my_ext_table_stream on external table my_ext_table insert_only = true; +``` + + + ![image.png](https://images.gitee.com/uploads/images/2021/0826/104137_439fc6d3_8136809.png) + +### reference + +[https://docs.snowflake.com/en/sql-reference/sql/create-stream.html](https://docs.snowflake.com/en/sql-reference/sql/create-stream.html) diff --git a/doc/basic-concepts/more-dataflow-samples/variable.md b/doc/basic-concepts/more-dataflow-samples/variable.md new file mode 100644 index 0000000..722b39c --- /dev/null +++ b/doc/basic-concepts/more-dataflow-samples/variable.md @@ -0,0 +1,87 @@ +### cursor, record variable + +This is an Oracle PLSQL. + +```sql +DECLARE + p_run_ind VARCHAR2; + TYPE acbal_cv IS REF CURSOR; + rec_dal_acbal T_DAL_ACBAL%ROWTYPE; +BEGIN + +IF p_run_ind = 'STEP1' THEN + OPEN acbal_cv FOR + SELECT product_type_code,product_code FROM T_DAL_ACBAL + WHERE AC_CODE > ' ' AND UPDT_FLG != '0' + AND UPDAT_FLG != '3' AND ROWNUM < 150001; + +ELSIF p_run_ind = 'STEP2' THEN + OPEN acbal_cv FOR + SELECT product_type_code,product_code FROM T_DAL_ACBAL + WHERE AC_CODE > ' ' AND UPDT_FLG != '0' + AND UPDAT_FLG != '3'; + +END IF; + +LOOP + FETCH acbal_cv INTO rec_dal_acbal; + EXIT WHEN cur_stclerk%NOTFOUND; + + UPDATE T_AC_MSTR + SET prd_type_code = rec_dal_acbal.product_type_code, + prd_code = rec_dal_acbal.product_code + ; + +END LOOP; + +COMMIT; +END; +``` + +#### dataflow in xml + +```xml + + + + + + + + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0705/110307_74539911_8136809.png) + +### scalar variable + +This is a Teradata stored procedure + +```sql +CREATE PROCEDURE NewProc (IN id CHAR(12), +IN pname INTEGER, +IN pid INTEGER, +OUT dname CHAR(10)) +BEGIN + + SELECT AGMT_ID + INTO dname FROM MY_EPRD2_VR_BASE.AGMT + WHERE PROCESS_ID = pid; +END; +``` + +#### dataflow in xml + +```xml + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0705/110824_924cc303_8136809.png) diff --git a/doc/basic-concepts/readme.md b/doc/basic-concepts/readme.md new file mode 100644 index 0000000..727bca6 --- /dev/null +++ b/doc/basic-concepts/readme.md @@ -0,0 +1,13 @@ +The basic concepts and elements of the data lineage + +### Concepts +- [1. Introduction](./1-introduction.md) +- [2. Direct dataflow](./2-direct-dataflow.md) +- [3. Indirect dataflow and pseudo column](./3-indirect-dataflow-and-pseudo-column.md) +- [4. Indirect dataflow: where clause and group by clause](./4-indirect-dataflow-where-group-by.md) +- [5. Dataflow: column used in aggregate function](./5-dataflow-column-used-in-aggregate-function.md) +- [6. Dataflow chain](./6-dataflow-chain.md) +- [7. Intermediate result set](./7-intermediate-resultset.md) +- [8. Join relation](./8-join-relation.md) +- [9. Temporary table](./9-temporary-table.md) +- [10. Transforms](./10-transforms.md) \ No newline at end of file diff --git a/doc/data lineage in multi queries.md b/doc/data lineage in multi queries.md new file mode 100644 index 0000000..6bb41a4 --- /dev/null +++ b/doc/data lineage in multi queries.md @@ -0,0 +1,66 @@ +The same column in different SQL statements can create different type column-level lineage. Those lineages should be picked up separately. + +```sql +CREATE VIEW dbo.hiredate_view(FirstName,LastName) +AS +SELECT p.FirstName, p.LastName +from Person.Person AS p +GO + +update dbo.hiredate_view h +set h.FirstName = p.FirstName +from h join Person.Person p +on h.id = p.id; + +insert into dbo.hiredate_view (FirstName,LastName) +SELECT p.FirstName, p.LastName +from Person.Person AS p ; +``` + +## column to column relations + +As you can see, the column: `FirstName` involves in the three SQL statements: create view, update and insert statement. + +While the column `LastName` involves in the two SQL statement: create view, insert statement. + +![image.png](https://images.gitee.com/uploads/images/2021/0709/171623_130af61b_8136809.png) + +**In the complete lineage mode**, if we turn off the `show intermediate recordset` option, you may find that although it gives you a higher level overview of the table to table relation, but some SQL statement related information such as how one column impact another column are missing. + +![image.png](https://images.gitee.com/uploads/images/2021/0709/172032_16b4e585_8136809.png) + +If we **check lineage in the table-level** via `table lineage` tab, you may find diagram like this: + +![image.png](https://images.gitee.com/uploads/images/2021/0709/172442_6c68a877_8136809.png) + +You can see that the statements that involved in the data transformation is persisted, but of course, since it's a table-level lineage, the columns involved in the lineage are hidden. So, it's your choice to use what's kind level of the lineage based on your requirements. + +## duplicated SQL query + +```sql +CREATE VIEW dbo.hiredate_view(FirstName,LastName) +AS +SELECT p.FirstName, p.LastName +from Person.Person AS p +GO + +update dbo.hiredate_view h +set h.FirstName = p.FirstName +from h join Person.Person p +on h.id = p.id; + +insert into dbo.hiredate_view (FirstName,LastName) +SELECT p.FirstName, p.LastName +from Person.Person AS p ; + +update dbo.hiredate_view h +set h.FirstName = p.FirstName +from h join Person.Person p +on h.id = p.id; +``` + +If the update statement is executed twice in the SQL batch as illustrated above, then you will see the update column-level lineage is showing twice in the diagram. These may not we want to see. + +![image.png](https://images.gitee.com/uploads/images/2021/0709/173001_64a0ade6_8136809.png) + +### how to avoid duplicate column-level lineage diff --git a/doc/data-lineage-analysis-command-line-tool.md b/doc/data-lineage-analysis-command-line-tool.md new file mode 100644 index 0000000..a3de3e0 --- /dev/null +++ b/doc/data-lineage-analysis-command-line-tool.md @@ -0,0 +1,97 @@ +## Data lineage analysis command line tool + +[Dlineage analysis command line tool](https://github.com/sqlparser/gsp_demo_java/releases) is used to analyze the data lineage in column level of a SQL query and output the data lineage in the XML/JSON format. + +By default, the tool will output the data lineage in the XML format and includes the most detailed information except the following elements which will be ignored: + +- the constant value (can be turned on using `/showConstant` option) +- transform: the transform code (can be turned on using `/transform` option) +- join: the join relation (can be turned on using `/j` option) + + +### Table level data lineage +This data lineage tool generate the data lineage at column level by default. +If you want to generate the data lineage at table level, you can use `/tableLineage` option. + +### Control the output of specific relation types +In the data lineage analysis, there are several types of relations, such as: +- fdd: [the direct data flow](./basic-concept/2-direct-dataflow.md) +- fdr: [the indirect data flow](./basic-concept/3-indirect-dataflow-and-pseudo-column.md) +- join: [the join relation](./basic-concept/8-join.md) +- call: [the call relation](./basic-cocept) +- er: [the entity-relationship relation](./basic-concept) + +You may want to control which relation types are included in the output. The `/includeRelationTypes` option allows you to specify which relation types should be included. The syntax is: + +``` +/includeRelationTypes fdd,fdr,join,call,er +``` + +Please note that relation type not specified in the `/includeRelationTypes` option will not be output. + + +### Parameters +#### /showResultSetTypes +There are many types of [the intermediate result sets](./basic-concepts/7-intermediate-resultset.md#intermediate-result-sets) will be generated during the analysis of the data lineage in SQL query. +When we ignore all the intermediate result sets (using `/s` option), the data lineage generated only includes the base tables and views. + +However, sometimes, we may want to see the data lineage of the intermediate result sets, for example, we want to see the data lineage of the result set generated by the aggregate function, or the result set generated by the insert/update clause in merge statement. + +In this case, we can use `/showResultSetTypes` option to specify the types of the result sets to be output. + +The result set types can be specified in the following format: + +``` +/showResultSetTypes array, struct, result_of, cte, insert_select, update_select, merge_update, merge_insert, output, update_set, pivot_table, unpivot_table, alias, rs, function, case_when +``` + +This option is valid only when `/s` option is used. + +#### /withTemporaryTable +Temporary table such as `#temp_table` in SQL Server will not be output in **the simple output** by default. +If you want to output the temporary table, you can use `/withTemporaryTable` option. + +``` +/s /withTemporaryTable +``` + +#### /j + +Option to output the join relation. +```sql +-- Oracle sample query +SELECT e.employee_id, e.first_name, e.last_name, d.department_name +FROM employees e +LEFT JOIN departments d ON e.department_id = d.department_id +ORDER BY e.employee_id; +``` + +#### call relation + +Option to output the call relation. +```sql +-- SQL Server sample query +CREATE PROCEDURE [dbo].[usp_insert] +( @a varchar(50), @b varchar(15), @c varchar(6), @d varchar(50) ) +AS +BEGIN + if ((select count(*) from tbl_Log1) <50000) + exec [dbo].[usp_insert_into_Log1] @a,@b,@c,@d + else + exec [dbo].[usp_insert_into_Log2] @a,@b,@c,@d +END +``` + +generated lineage with call relation: +```xml + + + + + + + + +``` + +#### er relation diff --git a/doc/data-lineage-model/data-lineage-format-introduction-1.png b/doc/data-lineage-model/data-lineage-format-introduction-1.png new file mode 100644 index 0000000..7aafa25 Binary files /dev/null and b/doc/data-lineage-model/data-lineage-format-introduction-1.png differ diff --git a/doc/data-lineage-model/data-lineage-format-introduction-2.png b/doc/data-lineage-model/data-lineage-format-introduction-2.png new file mode 100644 index 0000000..852209f Binary files /dev/null and b/doc/data-lineage-model/data-lineage-format-introduction-2.png differ diff --git a/doc/data-lineage-model/data-lineage-format-reference.md b/doc/data-lineage-model/data-lineage-format-reference.md new file mode 100644 index 0000000..9484998 --- /dev/null +++ b/doc/data-lineage-model/data-lineage-format-reference.md @@ -0,0 +1,111 @@ +# The reference of data lineage format + +## 1. Top level elements + +- dbvendor: string, SQL dialects such as oracle, mysql +- dbobjs: array, database objects such as table, view and etc +- relations: array, column-level relationship + +## 2. dbobjs + +This is an array that includes all database objects discovered during the data lineage analysis. JSON PATH: `$.dbobjs`. + +### Attributes + +* id, this is the unique identifier of this object in the data lineage result. +* name, name of the object. + +- type, type of the object such as table, view, process. +- coordinates, position of the object in the SQL script. +- columns, optional, valid when type is table, view + +## 3. realtions + +Relation is the atom unit of the data lineage. Relation build a link between the source and target column( column-level lineage). + +Those relations are stored in the `$.relations`. + +A relation includes the `type`, `target`, `sources` and other attributes. + +### Attributes + +- id +- type +- effectType +- target, is an object +- sources, array of source objects +- processid, optional + +## List of the elements in data lineage + +### 1. Database object + +- schema + +### 2. Schema object + +- table, including external table +- view +- procedure +- function +- trigger + +### 3. Hdfs object + +- file, such as `/user/data/pv_2008-06-08_us.txt, gs://bucket/path1.csv` +- directory, such as `s3://load/encrypted_files/`, `@exttable_part_stage/logs/` +- stage, this is a directory. + +### 4. SQL object + +Object created temporary during the execution of the SQL statement. + +- select list +- merge insert +- merge update +- update set +- update select +- insert select +- function call +- union +- CTE +- pivot table +- snowflake pivot alias +- sql server open json +- sql server join property +- set of rows ( constructed through the values expression, used in the place where table occurs) +- process, this the the SQL query that does the data transform. + +### 5. Relation + +* fdd , dataflow +* fdr , data impact + +### 6. join + +### 7. Abbreviation of the object + +The abbreviation of the object used in the diagram generated by the Gudu SQLFlow. + +- CTE [C] +- directory [D] +- function [F] +- function call [F] +- file [FL] +- insert select [IS] +- join property [JP] (sql server) +- merge insert [MI] +- merge update [MU] +- open json [OJ] (sql server) +- pivot alias [PA] (snowflake) +- pivot table [PT] +- procedure [P] +- select list [L] +- set of rows [RW] +- stage [S] +- table [T] +- trigger [TR] +- union [U] +- update set [UE] +- update select [US] +- view [V] \ No newline at end of file diff --git a/doc/data-lineage-model/elements-in-json/Introduction to the data lineage output.md b/doc/data-lineage-model/elements-in-json/Introduction to the data lineage output.md new file mode 100644 index 0000000..359fe4e --- /dev/null +++ b/doc/data-lineage-model/elements-in-json/Introduction to the data lineage output.md @@ -0,0 +1,109 @@ +This article gives you a basic idea about the main elements in a data lineage result generated by the Gudu SQLFlow in the JSON/XML format. + +For more detailed explanation about the elements used in a data lineage result, please check the data lineage structure reference after reading this article. + +## 1. Sample SQL + +We use a simple Oracle SQL query to demostrate the data lineage analysis result. + +```sql +INSERT INTO deptsal + (dept_no, + dept_name, + salary) +SELECT d.deptno, + d.dname, + SUM(e.sal + Nvl(e.comm, 0)) AS sal +FROM dept d + left join (SELECT * + FROM emp + WHERE hiredate > DATE '1980-01-01') e + ON e.deptno = d.deptno +GROUP BY d.deptno, + d.dname; +``` + +## 2. Data lineage diagram + +The data lineage generated by the Gudu SQLFlow for the above SQL. + +![image.png](https://images.gitee.com/uploads/images/2021/1118/113232_96df82d0_8136809.png) + +## 3. JSON output of this data lineage + +Link to the JSON file. + +## Main elements in the result + +### 3.1 Database objects, JSON path: $.dbobjs + +All database objects discovered during the data lineage analysis are stored in the `$.dbobjs` object. + +#### Table + +In the above sample SQL, there are four tables founded: + +- DEPTSAL + you can use `$.dbobjs[1].name` to return the table name, and `$.dbobjs[1].type` to return the type of this object which is `table` in this case. + you can also use expression like this to get this table: + ``` + $.dbobjs[?(@.name=='deptsal')].name + ``` +- DEPT + ``` + $.dbobjs[?(@.name=='dept')].name + ``` +- EMP + ```json + $.dbobjs[?(@.name=='emp')].name + ``` +- SQL_CONSTANTS + This is not a real table, but a table generated by the Gudu SQLFlow to store the constant used in the SQL query. + ```json + $.dbobjs[?(@.name=='SQL_CONSTANTS')].name + ``` + +### 3.2 Relation, JSON path: $.relations + +Relation is the atom unit of the data lineage. Relation build a link between the source and target column( column-level lineage). + +Those relations are stored in the `$.relations`. + +A relation includes the `type`, `target`, `sources` and other attributes. + +- type + There are 2 types of relation between source and target column. + a) Direct dataflow: represented by the `fdd` in the JSON output. + b) In-direct dataflow: represented by the `fdr` in the JSON output. In-direct dataflow also known as `impact` type, which means the value of the source column doesn't affect the value of the target column, but effect the rows number of the target column. For instance, the relation between a source column in the where clause and the column in the select list is a In-direct relation(impact). +- target + This is the target column. +- sources + These are source columns where the data of the target column comes from. + +#### Return a relation which target column is dept_no + +```json +$.relations[?(@.target.column=='dept_no')] +``` + +### 3.3 Connect relations to form a dataflow + +![image.png](https://images.gitee.com/uploads/images/2021/1119/100821_8c45ef71_8136809.png) + +Let's say you want to trace back from `DEPTSAL.dept_no` to `DEPT.deptno` in the above diagram. + +- Discover the relation 1 using the following JSON path + ```json + $.relations[?(@.target.column=='dept_no')] + ``` +- Discover the relation 2 using the following JSON path + ```json + $.relations[?(@.target.column=='deptno')] + ``` + + +## Summary + +This is a brief introduction about the structure and elements in a data lineage result generated by the Gudu SQLFlow. + +Please check the data lineage structure reference for more detailed information. diff --git a/doc/data-lineage-model/elements-in-json/Structure of lineage output file.md b/doc/data-lineage-model/elements-in-json/Structure of lineage output file.md new file mode 100644 index 0000000..10aec23 --- /dev/null +++ b/doc/data-lineage-model/elements-in-json/Structure of lineage output file.md @@ -0,0 +1,109 @@ +## 1. Top level elements + +- dbvendor: string, SQL dialects such as oracle, mysql +- dbobjs: array, database objects such as table, view and etc +- relations: array, column-level relationship + +## 2. dbobjs + +This is an array that includes all database objects discovered during the data lineage analysis. JSON PATH: `$.dbobjs`. + +### Attributes + +* id, this is the unique identifier of this object in the data lineage result. +* name, name of the object. + +- type, type of the object such as table, view, process. +- coordinates, position of the object in the SQL script. +- columns, optional, valid when type is table, view + +## 3. realtions + +Relation is the atom unit of the data lineage. Relation build a link between the source and target column( column-level lineage). + +Those relations are stored in the `$.relations`. + +A relation includes the `type`, `target`, `sources` and other attributes. + +### Attributes + +- id +- type +- effectType +- target, is an object +- sources, array of source objects +- processid, optional + +## List of the elements in data lineage + +### 1. Database object + +- schema + +### 2. Schema object + +- table, including external table +- view +- procedure +- function +- trigger + +### 3. Hdfs object + +- file, such as `/user/data/pv_2008-06-08_us.txt, gs://bucket/path1.csv` +- directory, such as `s3://load/encrypted_files/`, `@exttable_part_stage/logs/` +- stage, this is a directory. + +### 4. SQL object + +Object created temporary during the execution of the SQL statement. + +- select list +- merge insert +- merge update +- update set +- update select +- insert select +- function call +- union +- CTE +- pivot table +- snowflake pivot alias +- sql server open json +- sql server join property +- set of rows ( constructed through the values expression, used in the place where table occurs) +- process, this the the SQL query that does the data transform. + +### 5. Relation + +* direct dataflow (~~fdd~~) +* indirect dataflow (~~fdr~~) + +### 6. join + +### 7. Abbreviation of the object + +The abbreviation of the object used in the diagram generated by the Gudu SQLFlow. + +- CTE [C] +- directory [D] +- function [F] +- function call [F] +- file [FL] +- insert select [IS] +- join property [JP] (sql server) +- merge insert [MI] +- merge update [MU] +- open json [OJ] (sql server) +- pivot alias [PA] (snowflake) +- pivot table [PT] +- procedure [P] +- select list [L] +- set of rows [RW] +- stage [S] +- table [T] +- trigger [TR] +- union [U] +- update set [UE] +- update select [US] +- view [V] diff --git a/doc/data-lineage-model/export-metadata-and-lineage/Save metadata into an RDBMS database.md b/doc/data-lineage-model/export-metadata-and-lineage/Save metadata into an RDBMS database.md new file mode 100644 index 0000000..be875a9 --- /dev/null +++ b/doc/data-lineage-model/export-metadata-and-lineage/Save metadata into an RDBMS database.md @@ -0,0 +1,88 @@ +There are 2 tables created in the database in order to store the metadata. + +### sqlflow_dbobjects + +This table is used to store metadata of all database objects except column which is stored in `sqlflow_columns` table. + +#### fields + +1. guid, this is the unique identity number of the object +2. parent_id, the guid of the parent object. +3. qualified_name, the fully qualified object name +4. object_type, type of this object. +5. ddl, the SQL script used to define this object, such as create table statement. +6. ddl_hashId, hash code of the ddl to unique identify a ddl +7. comment, comment about this object + +qualified_name should be unique in the same object_type. + +So, there is a unique key of this table: (qualified_name, object_type). + +Available value for object_type is: cluster, database, table, view, column, procedure, function, trigger. + +The following predefined rows should be insert into this table: + +```sql +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('101','1','sqldialect','bigquery'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('102','1','sqldialect','couchbase'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('103','1','sqldialect','dax'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('104','1','sqldialect','db2'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('105','1','sqldialect','exasol'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('106','1','sqldialect','greenplum'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('107','1','sqldialect','hana'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('108','1','sqldialect','hive'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('109','1','sqldialect','impala'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('121','1','sqldialect','informix'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('122','1','sqldialect','mdx'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('123','1','sqldialect','mysql'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('124','1','sqldialect','netezza'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('125','1','sqldialect','odbc'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('126','1','sqldialect','openedge'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('127','1','sqldialect','oracle'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('128','1','sqldialect','postgresql'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('129','1','sqldialect','redshift'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('121','1','sqldialect','snowflake'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('122','1','sqldialect','sparksql'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('123','1','sqldialect','sqlserver'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('124','1','sqldialect','sybase'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('125','1','sqldialect','teradata'); +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('126','1','sqldialect','vertica'); +``` + +#### insert a new cluster + +insert a new hive cluster, with name `primary` and link to `hive` sql dialect which id is `108` + +```sql +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('1001','108','cluster','primary'); +``` + +#### insert a new database + +```sql +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('2001','1001','database','sampledb@primary'); +``` + +### insert a new table + +```sql +insert into sqlflow_dbobjects(guid,parent_id,object_type,qualified_name) values ('3001','2001','table','sampledb.tableA@primary'); +``` + +### sqlflow_columns + +This table is used to store all columns. + +#### fields + +1. guid, this is the unique identity number of the column +2. parent_id, the guid of the table which includes this column +3. qualified_name, the fully qualified object name +4. comment, comment about this column + +#### insert a column + +```sql +insert into sqlflow_dbobjects(guid,parent_id,qualified_name,comment) +values ('3001','2001','sampledb.tableA.columnB@primary','this is the comment'); +``` diff --git a/doc/data-lineage-model/export-metadata-and-lineage/export column-level lineage.md b/doc/data-lineage-model/export-metadata-and-lineage/export column-level lineage.md new file mode 100644 index 0000000..c1af012 --- /dev/null +++ b/doc/data-lineage-model/export-metadata-and-lineage/export column-level lineage.md @@ -0,0 +1,127 @@ +### column-level lineage + +The exported column-level linege should be in format like this: + +``` +source_db;source_schema;source_table;source_column;target_db;target_schema;target_table;target_column;procedure_names;query_hash_id +``` + +the exported column-level lineage **shouldn't include any intermediate recordset**, it only inclues the source and target table column, the hasd id of the query which does this transformation. + +#### sample sql + +```sql +CREATE VIEW dbo.hiredate_view(FirstName,LastName) +AS +SELECT p.FirstName, p.LastName +from Person.Person AS p +GO + +update dbo.hiredate_view h +set h.FirstName = p.FirstName +from h join Person.Person p +on h.id = p.id; + +insert into dbo.hiredate_view (FirstName,LastName) +SELECT p.FirstName, p.LastName +from Person.Person AS p ; +``` + +#### column-level lineage in xml + +```xml + + + + + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0709/180222_79498e1d_8136809.png) + +#### exported lineage + +```xml +source_db;source_schema;source_table;source_column;target_db;target_schema;target_table;target_column;procedure_names;query_hash_id +default;Person;Person;FirstName;default;default;hashId_of_update-set-1;FirstName;batchQueries;hashId_of_query +default;default;hashId_of_update-set-1;FirstName;default;dbo;hiredate_view;FirstName;batchQueries;hashId_of_query + +``` + +The column name in the exported lineage **shoudn't be qualified** , it must be like this `FirstName `. But when it is written to the data catalog such as Atlas, it must be qualified like this: `default.dbo.hiredate_view.FirstName.` + +The `hashId_of_update-set-1` is the pseduo name of the update-set resultset, it is MD5 value of string `type+column name in resultset`. + +The `hashId_of_query` is the MD5 value of the SQL query text from which this lineage is generated. + +Both of those hashId are used in order to make sure the resultset name or query from the same SQL statement is the same every time the SQL statement is executed. diff --git a/doc/data-lineage-model/export-metadata-and-lineage/export metadata to Atlas.md b/doc/data-lineage-model/export-metadata-and-lineage/export metadata to Atlas.md new file mode 100644 index 0000000..4f07f1c --- /dev/null +++ b/doc/data-lineage-model/export-metadata-and-lineage/export metadata to Atlas.md @@ -0,0 +1 @@ +.keep \ No newline at end of file diff --git a/doc/data-lineage-model/export-metadata-and-lineage/export metadata.md b/doc/data-lineage-model/export-metadata-and-lineage/export metadata.md new file mode 100644 index 0000000..1337768 --- /dev/null +++ b/doc/data-lineage-model/export-metadata-and-lineage/export metadata.md @@ -0,0 +1,117 @@ +The source of metadata comes from 2 sources, one is extracted from the database, the other is collect from the SQL scripts. + +In order to export the metadata collected by the SQLFlow, we need to know the structure that how SQLFlow save the + +metadata in the data lineage model. + +### 1. The database objects that need to be exported + +1. cluster +2. db +3. table/view +4. column +5. process, this is usually the query that transform the data, such as a stored proceudure, an insert statement and etc. + +#### defualt value for db and schema name + +If a db or schema is not mentioned in a SQL query, we use `default` as the name of the missing db or schema. + +The default schema for SQL Server database is `dbo`. + +### metadata from the database + +SQLFlow use the grabit tool to extract metadata from a database instance and save it in the format defined in this document. + +https://e.gitee.com/gudusoft/docs/591884/file/1434789?sub_id=4091727 + +#### cluster + +The related element in the exported json file is: `physicalInstance` + +#### db + +The related element in the exported json file is: `databases` + +#### table/view + +The related element in the exported json file is: `databases->tables` + +#### column + +The related element in the exported json file is: `databases->tables->column` + +### 2. metadata from the SQL script + +There is no `cluster` and `db` information in the json file generated by the SQLFlow after analyzing the SQL script and stored procedure. + +The database objects is saved in a json array `dbobjs` with the `name` and `type` property. + +```json +{ + "dbvendor": "dbvoracle", + "dbobjs": [ + { + "id": "37", + "name": "Query Create View", + "type": "process" + }, + { + "id": "4", + "schema": "scott", + "name": "scott.emp", + "type": "table", + "columns": [ + { + "id": "41", + "name": "deptno" + }, + { + "id": "42", + "name": "sal" + }, + { + "id": "43", + "name": "city" + } + ] + } + ] +} + +``` + +#### table/view + +table/view can be located via `dbobjs[index]` with the `type` set to `table`. + +#### column + +column can be located via `dbobjs[index]->columns[index]` + +### 3. uniquely identify a database object + +#### cluster + +1. Hive, the default cluster name is`primary` +2. Oracle, the cluster name is`physicalInstance` +3. SQL Server, the cluster name is the`servername` + +#### db + +The unique name of a database is in syntax like: `dbname@cluster`. Such as `ABCDPROD@xzy001db03.ddc.nba.com` + +#### table/view + +The unique name of a table is `dbname.tablename@cluster`, or `dbname.schemaname.tablename@cluster` . + +Such as `ABCDPROD.HARDWARE.SUBRACK_I_TRG@xzy001db03.ddc.nba.com` + +#### column + +The unique name of a column is `dbname.tablename.columnname@cluster`, or `dbname.schemaname.tablename.columnname@cluster` . + +#### process + +The unique name of a process is `dbname.procedureName.queryHashId@cluster`. + +The `proedureName` should be fully qualified. If the SQL query is not inside a stored proceure, then `procedureName` will use `batchQueries` as it name. `queryHashId` is the hash code of the SQL query that do the transformation, The `queryHashId` can be generated by calling the GSP library via `TParseTreeNode.getMd5()` method. diff --git a/doc/data-lineage-model/export-metadata-and-lineage/export table-level lineage.md b/doc/data-lineage-model/export-metadata-and-lineage/export table-level lineage.md new file mode 100644 index 0000000..fee5d3f --- /dev/null +++ b/doc/data-lineage-model/export-metadata-and-lineage/export table-level lineage.md @@ -0,0 +1,60 @@ +### table-level lineage + +The exported table-level linege should be in format like this: + +``` +source_db;source_schema;source_table;target_db;target_schema;target_table;procedure_names;query_hash_id +``` + +#### sample sql + +```sql +CREATE VIEW dbo.hiredate_view +AS +SELECT p.FirstName, p.LastName, e.BusinessEntityID, e.HireDate +FROM HumanResources.Employee e +JOIN Person.Person AS p ON e.BusinessEntityID = p.BusinessEntityID ; +``` + +#### lineage in XML + +```xml + + + + +
+ + + + + + + + + + + + + + + + + + +``` + +#### diagram + +![image.png](https://images.gitee.com/uploads/images/2021/0709/145303_36edb228_8136809.png) + +#### exported lineage + +```xml +source_db;source_schema;source_table;target_db;target_schema;target_table;procedure_names;query_hash_id +default;person;person;default;dbo;hiredate_view;batchQueries;xxxxx +default;HumanResources;Employee;default;dbo;hiredate_view;batchQueries;xxxxx + +``` + +The table name in the exported lineage **shoudn't be qualified** , it must be like this `Employee`. But when it is written to the data catalog such as Atlas, it must be qualified like this: `default.HumanResources.Employee` diff --git a/doc/data-lineage-model/models/Lineage model elements on UI.md b/doc/data-lineage-model/models/Lineage model elements on UI.md new file mode 100644 index 0000000..c304713 --- /dev/null +++ b/doc/data-lineage-model/models/Lineage model elements on UI.md @@ -0,0 +1,162 @@ +lineage model elements on UI + +## Entity + +path in the json: `data->sqlflow->dbobjs` + +### 1. Permanent entity + +#### 1. table + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/191609_0668124e_8136809.png "屏幕截图.png") + +#### 2. external table + +#### 3. view + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/191722_f1de58a0_8136809.png "屏幕截图.png") + +#### 4. hive local directory/inpath (type is path) + +```sql +LOAD DATA INPATH '/user/data/pv_2008-06-08_us.txt' INTO TABLE page_view PARTITION(date='2008-06-08', country='US') +``` + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0620/001315_8bb5cd3c_8136809.png "屏幕截图.png") + +#### 5. snowflake stage and path + +```sql +create or replace stage exttable_part_stage + url='s3://load/encrypted_files/' + credentials=(aws_key_id='1a2b3c' aws_secret_key='4x5y6z') + encryption=(type='AWS_SSE_KMS' kms_key_id = 'aws/key'); + +create external table exttable_part( + date_part date as to_date(split_part(metadata$filename, '/', 3) + || '/' || split_part(metadata$filename, '/', 4) + || '/' || split_part(metadata$filename, '/', 5), 'YYYY/MM/DD'), + timestamp bigint as (value:timestamp::bigint), + col2 varchar as (value:col2::varchar)) + partition by (date_part) + location=@exttable_part_stage/logs/ + auto_refresh = true + file_format = (type = parquet); + +``` + +![image.png](https://images.gitee.com/uploads/images/2021/0727/180953_b6445883_8136809.png) + +#### 6. bigquery file uri(type is path) + +```sql +CREATE EXTERNAL TABLE dataset.CsvTable OPTIONS ( + format = 'CSV', + uris = ['gs://bucket/path1.csv', 'gs://bucket/path2.csv'] +); +``` + +BigQuery create external table: +![输入图片说明](https://images.gitee.com/uploads/images/2021/0620/000802_4c090d22_8136809.png "屏幕截图.png") + +### 2. temporary entity + +#### 1. select_list + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/191530_b174d7df_8136809.png "屏幕截图.png") + +#### 2. merge_insert + +```sql +-- bigquery sample SQL +MERGE dataset.DetailedInventory T +USING dataset.Inventory S +ON T.product = S.product +WHEN NOT MATCHED AND s.quantity < 20 THEN + INSERT(product, quantity, supply_constrained, comments) + VALUES(product, quantity, true, ARRAY>[(DATE('2016-01-01'), 'comment1')]) +WHEN NOT MATCHED THEN + INSERT(product, quantity, supply_constrained) + VALUES(product, quantity, false) +; +``` + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/191107_8968ae0a_8136809.png "屏幕截图.png") + +#### 3. merge_update + +```sql +-- couchbase +MERGE INTO all_empts a USING emps_deptb b ON KEY b.empId +WHEN MATCHED THEN + UPDATE SET a.depts = a.depts + 1, + a.title = b.title || ", " || b.title +WHEN NOT MATCHED THEN + INSERT { "name": b.name, "title": b.title, "depts": b.depts, "empId": b.empId, "dob": b.dob } +``` + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/192035_de80e094_8136809.png "屏幕截图.png") + +#### 4. update_set + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/213231_ac0d23ec_8136809.png "屏幕截图.png") + +#### 5. update-select + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/213021_f18c731d_8136809.png "屏幕截图.png") + +#### 6. insert-select + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/213658_6300b209_8136809.png "屏幕截图.png") + +#### 7. function + +In order to show the function in the result, please turn on this setting: + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/234638_578c62bb_8136809.png "屏幕截图.png") + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/234222_d5dbc796_8136809.png "屏幕截图.png") + +#### 8. union + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/214736_2bd5f7e1_8136809.png "屏幕截图.png") + +#### 9. cte + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/214239_8bd93fc3_8136809.png "屏幕截图.png") + +#### 10. pivot table + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0619/235133_d683d625_8136809.png "屏幕截图.png") + +#### 11. snowflake pivot alias + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/200339_c6e02fef_8139001.png "屏幕截图.png") + +#### 12. mssql open json + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/201042_fa146c15_8139001.png "屏幕截图.png") + +#### 13. mssql json property + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/201357_0b1297e2_8139001.png "屏幕截图.png") + +## relationship + +path in the json: `data->sqlflow->relations` + +#### 1. fdd, data flow + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/192123_cd45caaf_8136809.png "屏幕截图.png") + +#### 2. fdr, frd data impact + +dash line + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/192342_469b7474_8136809.png "屏幕截图.png") + +#### 3. join + +dash line + +![输入图片说明](https://images.gitee.com/uploads/images/2021/0614/201812_e60c597c_8139001.png "屏幕截图.png") diff --git a/doc/data-lineage-model/models/Objects in data lineage model.md b/doc/data-lineage-model/models/Objects in data lineage model.md new file mode 100644 index 0000000..d60cafe --- /dev/null +++ b/doc/data-lineage-model/models/Objects in data lineage model.md @@ -0,0 +1,120 @@ +最顶层的对象为 database。模型中包含的对象分为这几类： database object, schema object, hdfs object, SQL object。 + +模型中包含的关系为 relation, join 。 + +在数据血缘模型中，主要为实体、属性和处理者（处理者可以看成是一种特殊类型的实体）。实体包含多个属性。处理者进行数据转换，可进行实体级的数据转换，也可以进行属性级的数据转换。 + +在RDBMS中，表，视图为实体，表中的字段称为表这个实体的属性。create view 等语句称为处理者。表级的数据血缘就是SQL语句对表进行数据转换。 + +字段级的数据血缘就是SQL 表达式对数据进行转换。 + +在HDFS中，文件为实体，文件中的字段称为这个文件的属性。 + +数据血缘模型包含两个主要的组成部分：实体和属性间的关系，用dbobjs来存储实体，用relations来存储属性间的关系。 + +最为典型的例子是dbobjs中存储的表，relations中存储的字段间的关系。 + +![image.png](https://images.gitee.com/uploads/images/2021/1022/154144_2642840e_8136809.png) + +## 基础数据血缘模型 + +relations中存储属性间的关系没有合并，没有形成完整的数据流。 + +目前只支持从 SQL 脚本生成该模型。以后需逐渐添加分析其它数据源，生成该模型。可以复用该模型之后路径中的所有功能。 + +参考 apache atlas type system: https://atlas.apache.org/#/TypeSystem + +![image.png](https://images.gitee.com/uploads/images/2021/1022/175019_c1d39fab_8136809.png) + +为了支持从不同的源头采集数据血缘模型，对该模型包含的内容和格式做如下的约定： + +### 模型中的实体 + +所有进入该模型的数据首先抽象为实体，例如RDBMS中的表。SQL语句也可以抽象为一个特殊类型的实体，即处理者。实体可以包含属性，例如表包含字段。 + +实体间可以建立关系，属性间也可以建立关系。 + +### 模型中的关系 + +关系分为两级，实体间的关系，例如RDBMS中的表直接的关系。属性间的关系，例如RDBMS中的字段间的关系。 + +一个关系包含一个目标对象和一个或多个源对象，还包含一个关系的类型，fdd: 实际的数据流。fdr: 影响数据流，impact。 + +### 模型中数据处理者 + +数据处理者是一种特殊的实体，它用来对数据实体进行数据转换。例如，一个 create view 语句就是一个处理者实体。一个SQL语句处理实体可以包含表达式处理者属性，就象表包含字段一样。表达式处理者一般进行字段级别的数据转换。 + +## 合并后的数据血缘模型 + +对基础模型中的relations中存储属性间的关系进行合并，形成完整的数据流。 + +## 1. Database object + +- schema + +## 2. Schema object + +- table, including external table +- view +- procedure +- function +- trigger + +## 3. Hdfs object + +- file, such as `/user/data/pv_2008-06-08_us.txt, gs://bucket/path1.csv` +- directory, such as `s3://load/encrypted_files/`, `@exttable_part_stage/logs/` +- stage, this is a directory. + +## 4. SQL object + +Object created temporary during the execution of the SQL statement. + +- select list +- merge insert +- merge update +- update set +- update select +- insert select +- function call +- union +- CTE +- pivot table +- snowflake pivot alias +- sql server open json +- sql server join property +- set of rows ( constructed through the values expression, used in the place where table occurs) +- process, this the the SQL query that does the data transform. + +## 5. Relationship + +* direct(fdd) and indirect dataflow + +## 6. join + +## 7. Abbreviation of the object + +The abbreviation of the object used in the diagram generated by the Gudu SQLFlow. + +- CTE [C] +- directory [D] +- function [F] +- function call [F] +- file [FL] +- insert select [IS] +- join property [JP] (sql server) +- merge insert [MI] +- merge update [MU] +- open json [OJ] (sql server) +- pivot alias [PA] (snowflake) +- pivot table [PT] +- procedure [P] +- select list [L] +- set of rows [RW] +- stage [S] +- table [T] +- trigger [TR] +- union [U] +- update set [UE] +- update select [US] +- view [V] diff --git a/doc/data-lineage-model/models/lineage model - column-process.md b/doc/data-lineage-model/models/lineage model - column-process.md new file mode 100644 index 0000000..50074d7 --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - column-process.md @@ -0,0 +1,92 @@ +## column-process + +This is the business rule that convert column. + +Struct definition + +```json +{ + "elementName" : "column-process", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "dependency", + "typeName": "string", + "isOptional": false + }, + { + "name": "targetColumn", + "typeName": "integer", + "isOptional": false + }, + { + "name": "sourceColumns", + "typeName": "array", + "isOptional": false + }, + { + "name": "reltions", + "typeName": "array", + "isOptional": false + }, + { + "name": "hashId", + "typeName": "string", + "isOptional": false + }, + { + "name": "parentProcessId", + "typeName": "int", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +name of the column process. + +### dependency + +The lineage also captures the kind of dependency, as listed below: + +* SIMPLE: output column has the same value as the input +* FUNCTION: output column is transformed by function. +* EXPRESSION: output column is transformed by some expression at runtime (for e.g. a Hive SQL expression) on the Input Columns. +* SCRIPT: output column is transformed by a user provided script. + +### targetColumn + +id of the target column. + +### sourceColumns + +array of the source column id. + +### relations + +The array of relations that made up of this column-process. + +### hashId + +The hashId is calculated based on the hashId of the relations that made up of this column-process. diff --git a/doc/data-lineage-model/models/lineage model - column.md b/doc/data-lineage-model/models/lineage model - column.md new file mode 100644 index 0000000..75f675e --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - column.md @@ -0,0 +1,48 @@ +## column + +struct definition + +```json +{ + "elementName" : "column", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": true + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +column name in the original SQL query. + +### type + +type of this column. + +### `coordinate` + +Indicates the positions of the occurences of the column in the SQL script. diff --git a/doc/data-lineage-model/models/lineage model - path.md b/doc/data-lineage-model/models/lineage model - path.md new file mode 100644 index 0000000..9f082ef --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - path.md @@ -0,0 +1,64 @@ +## path + +This is the path such as hdfs path, Amazon S3 path, BigQuery GS path. + +struct definition + +```json +{ + "elementName" : "path", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "uri", + "typeName": "string", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + }, + { + "name": "columns", + "typeName": "array", + "isOptional": true + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +the name of the path. + +### type + +type of the path, one of `hdfs`, Amazon `s3`, BigQuery `GS` + +### uri + +the path where the object is stored. + +### columns + +Path doesn't has columns in fact. We add columns here in order to make path available in column-level lineage model by using the pseudo column. diff --git a/doc/data-lineage-model/models/lineage model - procedure.md b/doc/data-lineage-model/models/lineage model - procedure.md new file mode 100644 index 0000000..a1c14f3 --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - procedure.md @@ -0,0 +1,95 @@ +## procedure + +Represents a stored procedure. + +struct definition + +```json +{ + "elementName" : "procedure", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + }, + { + "name": "arguments", + "typeName": "array", + "isOptional": true + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +procedure name in the original SQL query. + +### type + +One of those values: `createprocedure` + +### coordinate + +Indicates the positions of the occurrences in the SQL script. + +### argument + +argument of the stored procedure + +struct definition + +```json +{ + "elementName" : "argument", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "datatype", + "typeName": "string", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + }, + { + "name": "inout", + "typeName": "string", + "isOptional": true + } + ] +} +``` diff --git a/doc/data-lineage-model/models/lineage model - process.md b/doc/data-lineage-model/models/lineage model - process.md new file mode 100644 index 0000000..3183354 --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - process.md @@ -0,0 +1,75 @@ +## process + +This is the SQL statement that transforms the data. + +struct definition + +```json +{ + "elementName" : "process", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "queryHashId", + "typeName": "string", + "isOptional": false + }, + { + "name": "procedureName", + "typeName": "string", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +name of the process. + +### type + +type of the process, usually, it's the type of SQL statement that do the data transformation. Available value: + +* Create Table +* Create External Table +* Create View +* Create Stage +* Alter Table +* Update +* Merge +* Insert +* Select Into +* Hive Load + +### queryHashId + +This is the MD5 hash id that uniquely identify this SQL query. This `queryHashId` will be used when update a column or table-level lineage in the Atlas or other data catalog. + +### procedureName + +If this query statement is inside a stored procedure, this `procedureName` is the fully qualified name of the stored procedure. Otherwise, the `procedureName` should always be the `batchQueries` diff --git a/doc/data-lineage-model/models/lineage model - relation.md b/doc/data-lineage-model/models/lineage model - relation.md new file mode 100644 index 0000000..e7c6ff5 --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - relation.md @@ -0,0 +1,161 @@ +## relation + +Relation represents the column-level lineage. It includes **one target column, one or more source columns**. + +struct definition + +```json + + "elementName" : "relation", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "effectType", + "typeName": "string", + "isOptional": true + }, + { + "name": "processId", + "typeName": "string", + "isOptional": true + }, + { + "name": "dependency", + "typeName": "string", + "isOptional": true + }, + { + "name": "coordinate", + "typeName": "array", + "isOptional": true + }, + { + "name": "target", + "typeName": "targetElement", + "isOptional": false + }, + { + "name": "source", + "typeName": "array", + "isOptional": false + } + ] +} +``` + +### id + +unique id in the output. + +### type + +type of the column-lineage, available value: `fdd`, `fdr`, `join`. + +Please check [dbobjects_relationship](dbobjects_relationship.md) for the detailed information. + +### effectType + +This is the SQL statement that generate this relation. +Available values: `select, insert, update, merge_update, merge_insert, create_view, create_table, merge, delete, function, rename_table, swap_table, like_table, cursor,` `trigger,` `create_view` + +### dependency + +How this reltion convert source column to target column. One of those values: simple, function, expression. + +### coordinate + +The coordindate of SQL code the build this relation. + +### processId + +This is the SQL query that build this relation. + +### queryHashId + +Use `processId` instead. + +~~This is the hash code of the SQL query text from which this relation is generated. The `queryHashId` combined with target and source columns can be used to determine a unique relation in the lineage model. It's useful when export the lineage into the data catalog such as the Apache Atals to avoid the duplicated relation been inserted.~~ + +~~The SQL query with the same queryHashId is treated as the same query. This is usually happened when a SQL query been executed multi times.~~ + +## target,source element + +```json +{ + "elementName" : "target", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "column", + "typeName": "string", + "isOptional": false + }, + { + "name": "parent_id", + "typeName": "int", + "isOptional": false + }, + { + "name": "parent_name", + "typeName": "string", + "isOptional": false + }, + { + "name": "source", + "typeName": "string", + "isOptional": true + }, + { + "name": "clauseType", + "typeName": "string", + "isOptional": true + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": true + } + ] +} +``` + +### id + +the unique id in the output. + +### column + +The name of the column. + +There is a specific column name: `PseudoRows`, which represents the number of rows in the table/view/resultset. [Check here](dbobjects_relationship.md) for more information. + +### parent_id + +This is usually the id of a table that this columns belongs. + +### parent_name + +This is usually the name of a table that this columns belongs. + +### source + +If the value of source is `system`, this means the column doesn't comes from the SQL query. It's generated by SQLFlow. + +### clauseType + +Where this column comes from, such as where clause. diff --git a/doc/data-lineage-model/models/lineage model - resultset.md b/doc/data-lineage-model/models/lineage model - resultset.md new file mode 100644 index 0000000..30b6fc0 --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - resultset.md @@ -0,0 +1,69 @@ +## resultset + +This is the intermediate recordset generated during the process of SQL query such as a select list. + +struct definition + +```json +{ + "elementName" : "resultset", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "alias", + "typeName": "string", + "isOptional": true + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "hashId", + "typeName": "string", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": true + }, + { + "name": "columns", + "typeName": "array", + "isOptional": true + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +name of this resultset. + +### alias + +alias of this resultset. + +### type + +type of the reseultset, available value: `select_list`, `merge-insert`,`merge_update`,`update_set`,`update-select`,`insert-select`. + +### hashId + +The hashId is generated by calcualting the MD5 value of string: `type+name of columns` in the resultset. The hashId of a resultset is used when export a column-level lineage, it is used as a table name of the column in this resultset. diff --git a/doc/data-lineage-model/models/lineage model - table.md b/doc/data-lineage-model/models/lineage model - table.md new file mode 100644 index 0000000..ebe036d --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - table.md @@ -0,0 +1,177 @@ +## table + +Table type represents the table object in a relational database. + +It also represents the derived table such as function table. + +struct definition + +```json +{ + "elementName" : "table", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "alias", + "typeName": "string", + "isOptional": true + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "subType", + "typeName": "string", + "isOptional": false + }, + { + "name": "database", + "typeName": "string", + "isOptional": true + }, + { + "name": "schema", + "typeName": "string", + "isOptional": true + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": true + }, + { + "name": "processIds", + "typeName": "int", + "isOptional": true + }, + { + "name": "columns", + "typeName": "array", + "isOptional": true + } + ] +} +``` + +### id + +unique id in the output. + +### name + +table name in the original SQL query. + +### alias + +alias of the table in the original SQL query. + +### type + +type of the table, available value: `table`,`pseudoTable` + +- table + +This means a base table found in the SQL query. + +```sql +create view v123 as select a,b +from employee a, name b +where employee.id = name.id +``` + +```xml +
+``` + +- constantTable + + ```sql + update t1 set f1 =3 + ``` + + In order to link constant 3 to f1, we create a constant table with the name SQL_CONSTANTS includes a single column to save the constant. + + ```xml +
+ +
+ ``` +- pseudoTable + +Due to the lack of metadata information, some columns can't be linked to a table correctly. +Those columns will be assigned to a pseudo table with name: `pseudo_table_include_orphan_column`. +The type of this table is `pseudoTable`. + +In the following sample sql, columm `a`, `b` can't be linked to a specific table without enough information, +so a pseudo table with name `pseudo_table_include_orphan_column` is created to contain those orphan columns. + +```sql +create view v123 as +select a,b from employee a, name b where employee.id = name.id +``` + +```xml + + + +
+``` + +### subType + +In the most case of SQL query, the table used is a base table. +However, derived tables are also used in the from clause or other places. + +The `subType` property in the `table` element tells you what kind of the derived table this table is. + +Take the following sql for example, `WarehouseReporting.dbo.fnListToTable` is a function that +used as a derived table. So, the value of `subType` is `function`. + +Currently(GSP 2.2.0.6), `function` is the only value of `subType`. More value of `tableType` will be added in the later version +such as `JSON_TABLE` for JSON_TABLE. + +```sql +select entry as Account FROM WarehouseReporting.dbo.fnListToTable(@AccountList) +``` + +```xml + + +
+``` + +### database + +The database this table belongs to. + +### schema + +The schema this table belongs to. + +### coordinate + +Indicates the positions the table occurs in the SQL script. + +`coordinate="[1,37,0],[1,47,0]"` + +the first number is line， the second number is column, the third number is SQL script index of task. SqlInfoHelper uses the third number to position SQL script. + +### processIds + +The Id of the process which is doing the transformation related to this table. This `processIds` is used when generate table-level lineage model. + +### columns + +Array of `column` beblogs to this table. diff --git a/doc/data-lineage-model/models/lineage model - variable.md b/doc/data-lineage-model/models/lineage model - variable.md new file mode 100644 index 0000000..b4417ee --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - variable.md @@ -0,0 +1,64 @@ +## variable + +the variable used in the SQL especially in the stored procedure. + +struct definition + +```json +{ + "elementName" : "variable", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "subType", + "typeName": "string", + "isOptional": false + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": false + }, + { + "name": "columns", + "typeName": "array", + "isOptional": true + } + ] +} +``` + +### id + +the unique id in the output. + +### name + +variable name in the original SQL query. + +### type + +This value is always be `type` + +### subType + +type of the variable, one of those values: `scalar`,`cursor`,`record` + +### columns + +Array of column name in the cursor/record variable. Or the variable name of the scalar variable. diff --git a/doc/data-lineage-model/models/lineage model - view.md b/doc/data-lineage-model/models/lineage model - view.md new file mode 100644 index 0000000..a6c2e29 --- /dev/null +++ b/doc/data-lineage-model/models/lineage model - view.md @@ -0,0 +1,75 @@ +## view + +struct definition + +```json +{ + "elementName" : "view", + "attributeDefs": [ + { + "name": "id", + "typeName": "int", + "isOptional": false, + "isUnique": true + }, + { + "name": "name", + "typeName": "string", + "isOptional": false + }, + { + "name": "alias", + "typeName": "string", + "isOptional": true + }, + { + "name": "type", + "typeName": "string", + "isOptional": false + }, + { + "name": "database", + "typeName": "string", + "isOptional": true + }, + { + "name": "schema", + "typeName": "string", + "isOptional": true + }, + { + "name": "processIds", + "typeName": "int", + "isOptional": true + }, + { + "name": "coordinate", + "typeName": "string", + "isOptional": true + }, + { + "name": "columns", + "typeName": "array", + "isOptional": true + } + ] +} +``` + +### id + +unique id in the output. + +### name + +view name in the original SQL query. + +### alias + +alias of the view in the original SQL query. + +### type + +type of the view, available value: `view` + +### processIds diff --git a/doc/data-lineage-model/oracle-lineage.json b/doc/data-lineage-model/oracle-lineage.json new file mode 100644 index 0000000..df858ee --- /dev/null +++ b/doc/data-lineage-model/oracle-lineage.json @@ -0,0 +1,1719 @@ +{ + "dbvendor": "dbvoracle", + "dbobjs": [ + { + "id": "6", + "name": "Query Insert", + "procedureName": "batchQueries", + "queryHashId": "0141317811aa76e20b82376fac9294ac", + "type": "process", + "coordinates": [ + { + "x": 1, + "y": 1, + "hashCode": "0" + }, + { + "x": 14, + "y": 19, + "hashCode": "0" + } + ] + }, + { + "id": "2", + "name": "deptsal", + "type": "table", + "columns": [ + { + "id": "3", + "name": "dept_no", + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 21, + "hashCode": "0" + } + ] + }, + { + "id": "4", + "name": "dept_name", + "coordinates": [ + { + "x": 3, + "y": 14, + "hashCode": "0" + }, + { + "x": 3, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "5", + "name": "salary", + "coordinates": [ + { + "x": 4, + "y": 14, + "hashCode": "0" + }, + { + "x": 4, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "1", + "name": "PseudoRows", + "coordinates": [ + { + "x": 1, + "y": 13, + "hashCode": "0" + }, + { + "x": 1, + "y": 20, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 1, + "y": 13, + "hashCode": "0" + }, + { + "x": 1, + "y": 20, + "hashCode": "0" + } + ] + }, + { + "id": "8", + "name": "dept", + "alias": "d", + "type": "table", + "columns": [ + { + "id": "40", + "name": "deptno", + "coordinates": [ + { + "x": 12, + "y": 29, + "hashCode": "0" + }, + { + "x": 12, + "y": 37, + "hashCode": "0" + } + ] + }, + { + "id": "41", + "name": "dname", + "coordinates": [ + { + "x": 6, + "y": 8, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 8, + "y": 8, + "hashCode": "0" + }, + { + "x": 8, + "y": 14, + "hashCode": "0" + } + ] + }, + { + "id": "17", + "name": "emp", + "type": "table", + "columns": [ + { + "id": "18", + "name": "hiredate", + "coordinates": [ + { + "x": 11, + "y": 26, + "hashCode": "0" + }, + { + "x": 11, + "y": 34, + "hashCode": "0" + } + ] + }, + { + "id": "19", + "name": "deptno", + "coordinates": [ + { + "x": 12, + "y": 18, + "hashCode": "0" + }, + { + "x": 12, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "20", + "name": "sal", + "coordinates": [ + { + "x": 7, + "y": 12, + "hashCode": "0" + }, + { + "x": 7, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "21", + "name": "comm", + "coordinates": [ + { + "x": 7, + "y": 24, + "hashCode": "0" + }, + { + "x": 7, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "23", + "name": "*", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 10, + "y": 26, + "hashCode": "0" + }, + { + "x": 10, + "y": 29, + "hashCode": "0" + } + ] + }, + { + "id": "25", + "name": "SQL_CONSTANTS", + "type": "constantTable", + "columns": [ + { + "id": "26", + "name": "DATE '1980-01-01'", + "coordinates": [ + { + "x": 11, + "y": 37, + "hashCode": "0" + }, + { + "x": 11, + "y": 54, + "hashCode": "0" + } + ] + }, + { + "id": "38", + "name": "0", + "coordinates": [ + { + "x": 7, + "y": 32, + "hashCode": "0" + }, + { + "x": 7, + "y": 33, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": -1, + "y": -1, + "hashCode": "0" + }, + { + "x": -1, + "y": -1, + "hashCode": "0" + } + ] + }, + { + "id": "15", + "name": "RESULT_OF_E", + "type": "select_list", + "columns": [ + { + "id": "22", + "name": "*", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + { + "id": "42", + "name": "deptno", + "coordinates": [ + { + "x": 12, + "y": 18, + "hashCode": "0" + }, + { + "x": 12, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "14", + "name": "PseudoRows", + "coordinates": [ + { + "x": 11, + "y": 56, + "hashCode": "0" + }, + { + "x": 11, + "y": 57, + "hashCode": "0" + } + ], + "source": "system" + }, + { + "id": "22_0", + "name": "SAL", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + { + "id": "22_1", + "name": "COMM", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + { + "id": "22_2", + "name": "HIREDATE", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + } + ], + "coordinates": [ + { + "x": 11, + "y": 56, + "hashCode": "0" + }, + { + "x": 11, + "y": 57, + "hashCode": "0" + } + ] + }, + { + "id": "28", + "name": "INSERT-SELECT-1", + "type": "insert-select", + "columns": [ + { + "id": "29", + "name": "deptno", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 5, + "y": 16, + "hashCode": "0" + } + ] + }, + { + "id": "30", + "name": "dname", + "coordinates": [ + { + "x": 6, + "y": 8, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + { + "id": "31", + "name": "sal", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + }, + { + "id": "27", + "name": "PseudoRows", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + }, + { + "id": "33", + "name": "FUNCTION-1", + "type": "function", + "columns": [ + { + "id": "34", + "name": "SUM", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 11, + "hashCode": "0" + } + ] + }, + { + "id": "32", + "name": "PseudoRows", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 35, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 35, + "hashCode": "0" + } + ] + }, + { + "id": "36", + "name": "FUNCTION-2", + "type": "function", + "columns": [ + { + "id": "37", + "name": "Nvl", + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 23, + "hashCode": "0" + } + ] + }, + { + "id": "35", + "name": "PseudoRows", + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 34, + "hashCode": "0" + } + ], + "source": "system" + } + ], + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 34, + "hashCode": "0" + } + ] + } + ], + "relations": [ + { + "id": "1_0", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22_0", + "column": "SAL", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "20", + "column": "sal", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 7, + "y": 12, + "hashCode": "0" + }, + { + "x": 7, + "y": 17, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "1_1", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22_1", + "column": "COMM", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "21", + "column": "comm", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 7, + "y": 24, + "hashCode": "0" + }, + { + "x": 7, + "y": 30, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "1_2", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22_2", + "column": "HIREDATE", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "18", + "column": "hiredate", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 11, + "y": 26, + "hashCode": "0" + }, + { + "x": 11, + "y": 34, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "1_3", + "type": "fdd", + "effectType": "select", + "target": { + "id": "42", + "column": "DEPTNO", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 12, + "y": 18, + "hashCode": "0" + }, + { + "x": 12, + "y": 26, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "19", + "column": "deptno", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 12, + "y": 18, + "hashCode": "0" + }, + { + "x": 12, + "y": 26, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "1", + "type": "fdd", + "effectType": "select", + "target": { + "id": "22", + "column": "*", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "18", + "column": "hiredate", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 11, + "y": 26, + "hashCode": "0" + }, + { + "x": 11, + "y": 34, + "hashCode": "0" + } + ] + }, + { + "id": "19", + "column": "deptno", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 12, + "y": 18, + "hashCode": "0" + }, + { + "x": 12, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "20", + "column": "sal", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 7, + "y": 12, + "hashCode": "0" + }, + { + "x": 7, + "y": 17, + "hashCode": "0" + } + ] + }, + { + "id": "21", + "column": "comm", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 7, + "y": 24, + "hashCode": "0" + }, + { + "x": 7, + "y": 30, + "hashCode": "0" + } + ] + }, + { + "id": "23", + "column": "*", + "parentId": "17", + "parentName": "emp", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "4", + "type": "fdd", + "effectType": "select", + "target": { + "id": "29", + "column": "deptno", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 5, + "y": 16, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "40", + "column": "deptno", + "parentId": "8", + "parentName": "dept", + "coordinates": [ + { + "x": 12, + "y": 29, + "hashCode": "0" + }, + { + "x": 12, + "y": 37, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "5", + "type": "fdd", + "effectType": "select", + "target": { + "id": "30", + "column": "dname", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 6, + "y": 8, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "41", + "column": "dname", + "parentId": "8", + "parentName": "dept", + "coordinates": [ + { + "x": 6, + "y": 8, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "6", + "type": "fdd", + "effectType": "select", + "target": { + "id": "31", + "column": "sal", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "34", + "column": "SUM", + "parentId": "33", + "parentName": "FUNCTION-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 11, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "7", + "type": "fdd", + "effectType": "function", + "target": { + "id": "34", + "column": "SUM", + "parentId": "33", + "parentName": "FUNCTION-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "37", + "column": "Nvl", + "parentId": "36", + "parentName": "FUNCTION-2", + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 23, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "8", + "type": "fdd", + "effectType": "function", + "target": { + "id": "37", + "column": "Nvl", + "parentId": "36", + "parentName": "FUNCTION-2", + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "22_1", + "column": "COMM", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "10", + "type": "fdd", + "effectType": "function", + "target": { + "id": "37", + "column": "Nvl", + "parentId": "36", + "parentName": "FUNCTION-2", + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "38", + "column": "0", + "parentId": "25", + "parentName": "SQL_CONSTANTS", + "coordinates": [ + { + "x": 7, + "y": 32, + "hashCode": "0" + }, + { + "x": 7, + "y": 33, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "11", + "type": "fdd", + "function": "Nvl", + "effectType": "function", + "target": { + "id": "34", + "column": "SUM", + "parentId": "33", + "parentName": "FUNCTION-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "22_0", + "column": "SAL", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 9, + "y": 26, + "hashCode": "0" + }, + { + "x": 9, + "y": 27, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "17", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "3", + "column": "dept_no", + "parentId": "2", + "parentName": "deptsal", + "coordinates": [ + { + "x": 2, + "y": 14, + "hashCode": "0" + }, + { + "x": 2, + "y": 21, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "29", + "column": "deptno", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 5, + "y": 16, + "hashCode": "0" + } + ] + } + ], + "processId": "6" + }, + { + "id": "18", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "4", + "column": "dept_name", + "parentId": "2", + "parentName": "deptsal", + "coordinates": [ + { + "x": 3, + "y": 14, + "hashCode": "0" + }, + { + "x": 3, + "y": 23, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "30", + "column": "dname", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 6, + "y": 8, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + } + ], + "processId": "6" + }, + { + "id": "19", + "type": "fdd", + "effectType": "insert", + "target": { + "id": "5", + "column": "salary", + "parentId": "2", + "parentName": "deptsal", + "coordinates": [ + { + "x": 4, + "y": 14, + "hashCode": "0" + }, + { + "x": 4, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "31", + "column": "sal", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + } + ], + "processId": "6" + }, + { + "id": "13", + "type": "fdr", + "function": "SUM", + "effectType": "function", + "target": { + "id": "34", + "column": "SUM", + "parentId": "33", + "parentName": "FUNCTION-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "40", + "column": "deptno", + "parentId": "8", + "parentName": "dept", + "coordinates": [ + { + "x": 12, + "y": 29, + "hashCode": "0" + }, + { + "x": 12, + "y": 37, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "14", + "type": "fdr", + "function": "SUM", + "effectType": "function", + "target": { + "id": "34", + "column": "SUM", + "parentId": "33", + "parentName": "FUNCTION-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 11, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "41", + "column": "dname", + "parentId": "8", + "parentName": "dept", + "coordinates": [ + { + "x": 6, + "y": 8, + "hashCode": "0" + }, + { + "x": 6, + "y": 15, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "2", + "type": "fdr", + "effectType": "select", + "target": { + "id": "14", + "column": "PseudoRows", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 11, + "y": 56, + "hashCode": "0" + }, + { + "x": 11, + "y": 57, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "26", + "column": "DATE '1980-01-01'", + "parentId": "25", + "parentName": "SQL_CONSTANTS", + "coordinates": [ + { + "x": 11, + "y": 37, + "hashCode": "0" + }, + { + "x": 11, + "y": 54, + "hashCode": "0" + } + ] + }, + { + "id": "18", + "column": "hiredate", + "parentId": "17", + "parentName": "emp", + "clauseType": "where", + "coordinates": [ + { + "x": 11, + "y": 26, + "hashCode": "0" + }, + { + "x": 11, + "y": 34, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "3", + "type": "fdr", + "effectType": "select", + "target": { + "id": "27", + "column": "PseudoRows", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "14", + "column": "PseudoRows", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 11, + "y": 56, + "hashCode": "0" + }, + { + "x": 11, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "9", + "type": "fdr", + "target": { + "id": "35", + "column": "PseudoRows", + "parentId": "36", + "parentName": "FUNCTION-2", + "coordinates": [ + { + "x": 7, + "y": 20, + "hashCode": "0" + }, + { + "x": 7, + "y": 34, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "14", + "column": "PseudoRows", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 11, + "y": 56, + "hashCode": "0" + }, + { + "x": 11, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "12", + "type": "fdr", + "target": { + "id": "32", + "column": "PseudoRows", + "parentId": "33", + "parentName": "FUNCTION-1", + "coordinates": [ + { + "x": 7, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 35, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "14", + "column": "PseudoRows", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 11, + "y": 56, + "hashCode": "0" + }, + { + "x": 11, + "y": 57, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "15", + "type": "fdr", + "effectType": "select", + "target": { + "id": "27", + "column": "PseudoRows", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "42", + "column": "deptno", + "parentId": "15", + "parentName": "RESULT_OF_E", + "coordinates": [ + { + "x": 12, + "y": 18, + "hashCode": "0" + }, + { + "x": 12, + "y": 26, + "hashCode": "0" + } + ] + }, + { + "id": "40", + "column": "deptno", + "parentId": "8", + "parentName": "dept", + "clauseType": "joinCondition", + "coordinates": [ + { + "x": 12, + "y": 29, + "hashCode": "0" + }, + { + "x": 12, + "y": 37, + "hashCode": "0" + } + ] + } + ] + }, + { + "id": "16", + "type": "fdr", + "effectType": "insert", + "target": { + "id": "1", + "column": "PseudoRows", + "parentId": "2", + "parentName": "deptsal", + "coordinates": [ + { + "x": 1, + "y": 13, + "hashCode": "0" + }, + { + "x": 1, + "y": 20, + "hashCode": "0" + } + ] + }, + "sources": [ + { + "id": "27", + "column": "PseudoRows", + "parentId": "28", + "parentName": "INSERT-SELECT-1", + "coordinates": [ + { + "x": 5, + "y": 8, + "hashCode": "0" + }, + { + "x": 7, + "y": 42, + "hashCode": "0" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/doc/data-lineage-model/readme.md b/doc/data-lineage-model/readme.md new file mode 100644 index 0000000..5cbd089 --- /dev/null +++ b/doc/data-lineage-model/readme.md @@ -0,0 +1,108 @@ +This article gives you a basic idea about the main elements in a data lineage result generated by the Gudu SQLFlow in the JSON/XML format. + +For more detailed explanation about the elements used in a data lineage result, please check [the data lineage format reference](data-lineage-format-reference.md) after reading this article. + +## 1. Sample SQL + +We use a simple Oracle SQL query to demostrate the data lineage analysis result. + +```sql +INSERT INTO deptsal + (dept_no, + dept_name, + salary) +SELECT d.deptno, + d.dname, + SUM(e.sal + Nvl(e.comm, 0)) AS sal +FROM dept d + left join (SELECT * + FROM emp + WHERE hiredate > DATE '1980-01-01') e + ON e.deptno = d.deptno +GROUP BY d.deptno, + d.dname; +``` + +## 2. Data lineage diagram + +The data lineage generated by the Gudu SQLFlow for the above SQL. + +![image.png](data-lineage-format-introduction-1.png) + +## 3. JSON output of this data lineage + +[Link to the JSON file](oracle-lineage.json). + +## Main elements in the result + +### 3.1 Database objects, JSON path: $.dbobjs + +All database objects discovered during the data lineage analysis are stored in the `$.dbobjs` object. + +#### Table + +In the above sample SQL, there are four tables founded: + +- DEPTSAL + you can use `$.dbobjs[1].name` to return the table name, and `$.dbobjs[1].type` to return the type of this object which is `table` in this case. + you can also use expression like this to get this table: + ```json + $.dbobjs[?(@.name=='deptsal')].name + ``` +- DEPT + ```json + $.dbobjs[?(@.name=='dept')].name + ``` +- EMP + ```json + $.dbobjs[?(@.name=='emp')].name + ``` +- SQL_CONSTANTS + This is not a real table, but a table generated by the Gudu SQLFlow to store the constant used in the SQL query. + ```json + $.dbobjs[?(@.name=='SQL_CONSTANTS')].name + ``` + +### 3.2 Relation, JSON path: $.relations + +Relation is the atom unit of the data lineage. Relation build a link between the source and target column( column-level lineage). + +Those relations are stored in the `$.relations`. + +A relation includes the `type`, `target`, `sources` and other attributes. + +- type + There are 2 types of relation between source and target column. + a) Direct dataflow: represented by the `fdd` in the JSON output. + b) In-direct dataflow: represented by the `fdr` in the JSON output. In-direct dataflow also known as `impact` type, which means the value of the source column doesn't affect the value of the target column, but effect the rows number of the target column. For instance, the relation between a source column in the where clause and the column in the select list is a In-direct relation(impact). +- target + This is the target column. +- sources + These are source columns where the data of the target column comes from. + +#### Return a relation which target column is dept_no + +```json +$.relations[?(@.target.column=='dept_no')] +``` + +### 3.3 Connect relations to form a dataflow + +![image.png](data-lineage-format-introduction-2.png) + +Let's say you want to trace back from `DEPTSAL.dept_no` to `DEPT.deptno` in the above diagram. + +- Discover the relation 1 using the following JSON path + ```json + $.relations[?(@.target.column=='dept_no')] + ``` +- Discover the relation 2 using the following JSON path + ```json + $.relations[?(@.target.column=='deptno')] + ``` + +## Summary + +This is a brief introduction about the structure and elements in a data lineage result generated by the Gudu SQLFlow. + +Please check [the data lineage structure reference](data-lineage-format-reference.md) for more detailed information. \ No newline at end of file diff --git a/doc/readme.md b/doc/readme.md new file mode 100644 index 0000000..ae97278 --- /dev/null +++ b/doc/readme.md @@ -0,0 +1,36 @@ +## Gudu SQLFlow Documents + +Firstly, use [SQLFlow cloud](https://sqlflow.gudusoft.com/) to get the data lineage of your SQL and feel how the data lineage diagram can help you understand the data flow and the data transformation. + +Secondly, use [Dlineage command line tool](https://github.com/sqlparser/gsp_demo_java/releases) to get the data lineage of your SQL in the XML/JSON format and see how the data lineage is organized in the JSON/XML format and start to know various elements and attributes in the data lineage. + +Thirdly, check the basic concepts and elements of the data lineage. + +Fourthly, check the detailed reference of the data lineage model. + +Finally, integrate the SQLFlow Rest API/Java API/Widget into your project and get the data lineage of your SQL in the Rest API/Java/Python code/JavaScript code. + + +### Concepts +- [1. Introduction](basic-concepts/1-introduction.md) +- [2. Direct dataflow](basic-concepts/2-direct-dataflow.md) +- [3. Indirect dataflow and pseudo column](basic-concepts/3-indirect-dataflow-and-pseudo-column.md) +- [4. Indirect dataflow: where clause and group by clause](basic-concepts/4-indirect-dataflow-where-group-by.md) +- [5. Dataflow: column used in aggregate function](basic-concepts/5-dataflow-column-used-in-aggregate-function.md) +- [6. Dataflow chain](basic-concepts/6-dataflow-chain.md) +- [7. Intermediate result set](basic-concepts/7-intermediate-resultset.md) +- [8. Join relation](basic-concepts/8-join-relation.md) +- [9. Temporary table](basic-concepts/9-temporary-table.md) +- [10. Transforms](basic-concepts/10-transforms.md) + + +### Data lineage model (TODO) +- [1. Introduction](data-lineage-model/readme.md) +- [2. Detailed reference](data-lineage-model/data-lineage-model-reference.md) + +### The structure and elements of the data lineage result(JSON/XML). +- [1. Introduction](data-lineage-format/readme.md) +- [2. Detailed reference](data-lineage-format/data-lineage-format-reference.md) + +### SQLFlow widget +- [Introduction](widget/readme.md) \ No newline at end of file diff --git a/doc/sql-features/identifier-and-string-literal.md b/doc/sql-features/identifier-and-string-literal.md new file mode 100644 index 0000000..ffb94ce --- /dev/null +++ b/doc/sql-features/identifier-and-string-literal.md @@ -0,0 +1,252 @@ +## Identifier（标识符） and String literal（字符串常量） + +关于 Identifier（标识符） and String literal（字符串常量）详细介绍见： +https://www.dpriver.com/blog/2023/12/mastering-sql-syntax-a-comprehensive-guide-for-2024-and-beyond/ + +TODO: add String literal section in this article. + +这篇文章讨论一些具体问题。 + +## SQLServer + +### 常见的 Identifier 和 String literal 形式 +#### Identifier +- name +- [my name] +- "this is my name" + +#### String literal +- 'name' + + +### Column in select list +```sql + select name,[name],"name",'name', age from dbo.person +``` +上面这个语句中的 name,[name],"name" 都是 identifier, 而 'name' 为 string literal。 + +看下面的输出，identifier 会返回数据库表中对应字段的值，但 string literal 返回该字符串本身： +![images](identifier-and-string-literal01.png) + + +### Table alias +创建 table alias 时，Identifier 可以作为 table alias，但 string constant 不行 +```sql +-- 语法正确 +select name, [ps].age from dbo.person "ps" +select name, "ps".age from dbo.person [ps] +``` + +```sql +-- 语法错误 +select name, age from dbo.person 'ps' +``` + +### Column alias + +在创建 column alias 时，Identifier 和 string constant 都可以作为 column alias， +```sql +-- 语法正确 +select name name, name "first name", name "second name", name 'third name' from dbo.person +``` + +但在引用 column alias 时，仅能使用 identifier， 不能使用 string constant。 +```sql +-- 语法正确 +select p.[third name] from (select name name, name "first name", name [second name], name 'third name' from dbo.person) p +``` +**上面的这个 SQL 说明，subquery 中的 name 'third name' 和 外部的 p.[third name] 是指同一个 column，因此我们在比较时，仅比较 `third name`, 外面的 ' 和 [] 符号都需要去掉**。 + +```sql +-- 语法错误 +select p.'third name' from (select name name, name "first name", name [second name], name 'third name' from dbo.person) p +``` + +### 测试用的脚本 +```sql +use master +create table dbo.person(name varchar(100), age int ); +insert into dbo.person(name,age) values('Tom',11),('Alice',12); +create table dbo.student(sname varchar(100), sage int ); +insert into dbo.student(sname,sage) values('sTom',21),('sAlice',22); +``` + +## mysql + +#### Identifier + +- name +- \`name\` + +#### String literal + +- 'name' +- "name" + +### Column in select list： + +```sql +select name,`name`,"name",'name' from dbo.person +``` + +上面这个语句中的 name,\`name\` 都是 identifier, 而 'name',"name" 为 string literal。 + +### Table alias + +创建 table alias 时，Identifier 可以作为 table alias，但 string constant 不行，**且Identifier类型 无需保持一致** + +```sql +-- 语法正确 +select name, \`ps\`.age from dbo.person ps +select name, ps.age from dbo.person \`ps\` +``` + +```sql +-- 语法错误 +select name, age from dbo.person 'ps' +select name, age from dbo.person "ps" +``` + +### Column alias + +在创建 column alias 时，Identifier 和 string constant 都可以作为 column alias + +```sql +-- 语法正确 +select name name, name "first name", name 'second name', name \`third name\` from dbo.person +``` + +但在引用 column alias 时，仅能使用 identifier， 不能使用 string constant。**且Identifier 类型无需保持一致** + +```sql +-- 语法正确 +SELECT renta.staff FROM (SELECT rental_id "staff" FROM rental) `renta` +``` + + +## Oracle + +#### Identifier + +- name +- "name" + +#### String literal + +- 'name' + +### Column in select list： + +```sql +select name,"name",'name' from dbo.person +``` + +上面这个语句中的 name,"name" 都是 identifier, 而 'name', 为 string literal。 + +### Table alias + +创建 table alias 时，Identifier 可以作为 table alias，但 string constant 不行, **且Identifier类型 需要保持一致，不可混用** + +```sql +-- 语法正确 +select name, ps.age from dbo.person ps +select name, "ps".age from dbo.person "ps" +``` + +```sql +-- Identifier 语法错误 +select name, "ps".age from dbo.person ps +select name, ps.age from dbo.person "ps" +select name, 'ps'.age from dbo.person 'ps' +``` + +### Column alias + +在创建 column alias 时，Identifier 作为 column alias,但 string constant 不行 + +```sql +-- 语法正确 +select name name, name "first name" from dbo.person +``` + +在创建 column alias 时，Identifier 作为 column alias,但 string constant 不行 + +```sql +-- 语法错误 +select name 'name' from dbo.person +``` + +但在引用 column alias 时，仅能使用 identifier， 不能使用 string constant。**且Identifier 类型无需保持一致** + +```sql +-- 语法正确 +SELECT "P".REGION_ID2 FROM (SELECT REGION_ID "REGION_ID2" FROM REGIONS )P +``` + + + + + +## PostgreSQL + +#### Identifier + +- name +- "name" + +#### String literal + +- 'name' + +### Column in select list： + +```sql +select name,"name",'name' from dbo.person +``` + +上面这个语句中的 name,"name" 都是 identifier, 而 'name', 为 string literal。identifier和string literal都支持。 + +### Table alias + +创建 table alias 时，Identifier 可以作为 table alias，但 string constant 不行,**且Identifier类型 无需保持一致** + +```sql +-- 语法正确 +select name, "ps".age from dbo.person ps +select name, ps.age from dbo.person "ps" +``` + +```sql +-- Identifier 语法错误 +select name, 'ps'.age from dbo.person ps +``` + +### Column alias + +在创建 column alias 时，Identifier 作为 column alias,但 string constant 不行 + +```sql +-- 语法正确 +select name name, name "first name" from dbo.person +``` + +在创建 column alias 时，Identifier 作为 column alias,但 string constant 不行 + +```sql +-- 语法错误 +select name 'name' from dbo.person +``` + +但在引用 column alias 时，仅能使用 identifier， 不能使用 string constant。**且Identifier类型 需要保持一致，不可混用** + +```sql +-- 语法正确 +SELECT "P".REGION_ID2 FROM (SELECT column1 REGION_ID2 FROM newtable_1 ) "P" +select P."REGION_ID2" FROM (SELECT column1 "REGION_ID2" FROM newtable_1 ) P +``` + +```sql +-- 语法错误 +SELECT "P".REGION_ID2 FROM (SELECT column1 REGION_ID2 FROM newtable_1 ) P +select P."REGION_ID2" FROM (SELECT column1 REGION_ID2 FROM newtable_1 ) P +``` diff --git a/doc/sql-features/identifier-and-string-literal01.png b/doc/sql-features/identifier-and-string-literal01.png new file mode 100644 index 0000000..92aa67e Binary files /dev/null and b/doc/sql-features/identifier-and-string-literal01.png differ diff --git a/doc/sql-features/readme.md b/doc/sql-features/readme.md new file mode 100644 index 0000000..581d572 --- /dev/null +++ b/doc/sql-features/readme.md @@ -0,0 +1,6 @@ +Topics about SQL syntax of various databases. + +Generic SQL syntax will be put under this directory and database specific +syntax will be put under sub-directory with database name such as DB2, +Oracle and etc. + diff --git a/doc/sql-features/sqlserver/discover-data-lineage-from-subquery-and-CTE.md b/doc/sql-features/sqlserver/discover-data-lineage-from-subquery-and-CTE.md new file mode 100644 index 0000000..c6c1909 --- /dev/null +++ b/doc/sql-features/sqlserver/discover-data-lineage-from-subquery-and-CTE.md @@ -0,0 +1,85 @@ +# Discover data lineage from sub-query and CTE + +SQL is very flexible, and it's possible to write a SQL statement in many different ways to achieve the same result. + +The following SQL statement is used to update the PARENT_TABLE column of the CCF table based on the PARENT_TABLE column of the CCF_BAK table. +and it is written in 3 different ways. + +The first way is to use the UPDATE statement with a sub-query. +The second way is to use the UPDATE statement with 2 sub-queries. +The third way is to use the CTE (Common Table Expression) with the same result. + +SQLFlow can discover the data lineage from all the 3 SQL statements. +[![Discover data lineage from sub-query and CTE](../../../assets/images/discover-data-lineage-from-subquery-and-cte1.png)](https://sqlflow.gudusoft.com) + +```sql +-- sql server +UPDATE s +SET s.PARENT_TABLE = u.PARENT_TABLE +FROM dwh_user.jv.CCF s +JOIN ( + SELECT TOP 1 * + FROM dwh_user.jv.CCF_BAK u + WHERE u.PRODUCT_NAME = 'Business loan' +) u ON 1 = 1 +WHERE s.PRODUCT_NAME = 'Business loan' +``` + +Data lineage of the first SQL statement in xml format generated by [SQLFlow](https://sqlflow.gudusoft.com/): +```xml + + + + +``` + +```sql +-- sql server +UPDATE s +SET s.PARENT_TABLE = u.PARENT_TABLE +FROM ( + SELECT * + FROM dwh_user.jv.CCF s + WHERE s.PRODUCT_NAME = 'Business loan' +) s +JOIN ( + SELECT TOP 1 * + FROM dwh_user.jv.CCF_BAK u + WHERE u.PRODUCT_NAME = 'Business loan' +) u ON 1 = 1 +``` + +Data lineage of the second SQL statement in xml format generated by [SQLFlow](https://sqlflow.gudusoft.com/): +```xml + + + + +``` + +```sql +-- sql server +WITH + s AS ( + SELECT PARENT_TABLE + FROM dwh_user.jv.CCF s + WHERE s.PRODUCT_NAME = 'Business loan' + ), + u AS ( + SELECT TOP 1 PARENT_TABLE + FROM dwh_user.jv.CCF_BAK u + WHERE u.PRODUCT_NAME = 'Business loan' + ) +UPDATE s +SET s.PARENT_TABLE = u.PARENT_TABLE +FROM s +JOIN u ON 1 = 1 +``` + +Data lineage of the third SQL statement in xml format generated by [SQLFlow](https://sqlflow.gudusoft.com/): +```xml + + + + +``` diff --git a/doc/sql-features/sqlserver/return-records-from-proc.md b/doc/sql-features/sqlserver/return-records-from-proc.md new file mode 100644 index 0000000..00a2d5e --- /dev/null +++ b/doc/sql-features/sqlserver/return-records-from-proc.md @@ -0,0 +1,43 @@ +以下 SQL Server 的 SQL 语句通过执行 INSERT INTO TestTable(Column1, Column2) EXEC dbo.testPro，会将存储过程 dbo.testPro 返回的结果集插入到表 TestTable 中。 + +```sql +CREATE TABLE [dbo].[NewTable] ( + [Column1] varchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, + [Column2] varchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, + [Column3] varchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NULL +); + +CREATE TABLE [dbo].[NewTable2] ( + [Column1] varchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, + [Column2] varchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, + [Column3] varchar(100) COLLATE SQL_Latin1_General_CP1_CI_AS NULL +); + +CREATE PROCEDURE dbo.testPro +AS +BEGIN +SELECT Column1, Column2 FROM NewTable; +SELECT Column1, Column2 FROM NewTable2; +end; + +INSERT INTO TestTable(Column1, Column2) EXEC dbo.testPro; +``` + +存储过程 dbo.testPro 中有两个 SELECT 语句: +```sql +SELECT Column1, Column2 FROM NewTable; +SELECT Column1, Column2 FROM NewTable2; +``` + +因此，存储过程会返回 NewTable 和 NewTable2 两个表的 Column1 和 Column2 列的数据。这些数据会被插入到 TestTable 表的 Column1 和 Column2 列中。 + +SQL Server 的以上实现在其他数据库中不见得能正常工作，更通用的实现应该是： +```sql +CREATE PROCEDURE dbo.testPro +AS +BEGIN + SELECT Column1, Column2 FROM NewTable + union all + SELECT Column1, Column2 FROM NewTable2; +end; +``` \ No newline at end of file diff --git a/doc/the column-level lineage.md b/doc/the column-level lineage.md new file mode 100644 index 0000000..a70e9d1 --- /dev/null +++ b/doc/the column-level lineage.md @@ -0,0 +1,20 @@ +```sql +Insert into enrollment2.EnrollmentCW +SELECT + CASE WHEN dstf.LastName + ', ' + dstf.FirstName LIKE '%DEFAULT%' THEN '' + ELSE dstf.LastName + ', ' + dstf.FirstName + END AS AdvisorFullName, + CASE WHEN dstf.StaffEmail = 'Unknown' THEN '' + ELSE dstf.StaffEmail + END AS AdvisorEmail +FROM STU_DW.enrollment1.vwStudentEnrollmentCW fsecw + LEFT JOIN STU_DW.org.DimStaff dstf ON fsecw.DimStaffIDAdvisor = dstf.DimStaffID + AND dstf.IsCurrentRow = 1 +``` + + + ![image.png](https://images.gitee.com/uploads/images/2021/0712/183847_efe5301d_8136809.png) + +## 1. How to get column-level model from the complete model + +## 2. The export format of the column-level model diff --git a/doc/the complete data lineage.md b/doc/the complete data lineage.md new file mode 100644 index 0000000..1c2f0e6 --- /dev/null +++ b/doc/the complete data lineage.md @@ -0,0 +1,103 @@ +When analyzing data lineage, the complete model is always generated since all other higher level models are based on this model. + +## 1. Types of entity + +Table, view, column, process(SQL statement), resultset, function, variale, path. + +## 2. Types of relation + +### fdd + +The `fdd` relation means the data of the target entity comes from the source entity. Take this SQL query for example: + +```sql +SELECT a.empName "eName" +FROM scott.emp a +``` + +the data of target column `"eName"` comes from `scott.emp.empName`, so we have a dataflow relation like this: + +``` +scott.emp.empName -> fdd -> "eName" +``` + +![image.png](https://images.gitee.com/uploads/images/2021/0711/184154_5b0a97e7_8136809.png) + +the result generated by the select list is called: `resultset` which ikes a virtual table includes columns and rows. + +### fdr + +The `fdr` relation means the data of the source column will impact the row number of the resultset in the select list, or will impact the result value of an anggreate function. + +```sql +SELECT a.empName "eName" +FROM scott.emp a +Where sal > 1000 +``` + +The total number of rows in the select list is impacted by the value of column `sal` in the where clause. So we have a dataflow relation like this: + +``` +sal -> fdr -> resultset.PseudoRows +``` + +![image.png](https://images.gitee.com/uploads/images/2021/0711/184450_e112148b_8136809.png) + +#### PseudoRows column + +As you can see, we introduced a new pseudo column: `PseudoRows` to represents the number of rows in the resultset. + +The fdr type dataflow is represented by a dash line. You can hide the `fdr` type dataflow by turn off the `impact` option in the SQLFlow. + +![image.png](https://images.gitee.com/uploads/images/2021/0702/182855_81f59e19_8136809.png) + +You may find more examples about the `fdr` relation in the ***lineage in real SQL*** section. + +### join + +The `join` relation build a link between 2 or more columns in the join condition. Striclty speaking, the relation is not a dataflow relation. + +```sql + select b.teur + from tbl a left join TT b on (a.key=b.key) +``` + +A join relation will be created after analzye the above SQL. It indicates a join relation betwee `tbl.key` and `TT.key`. + +![image.png](https://images.gitee.com/uploads/images/2021/0711/185405_036c2a1a_8136809.png) + +## 3. Connect the entity using relation + +When build relation between 2 entities, the source and target entity can be column to column, or, table to table. + +### column to column + +This is the most often case that both entity in a relation are columns. + +### table to table + +Sometimes, there will be a relation between 2 tables. For example, in an alter table rename SQL statement, a table to table relation will be created. Acutally, a table to table relation is represented by a column to column relation using the `PseudoRows` column. + +Table to table relation is included in the complete lineage model by using `PseudoRows` for 2 reasons: + +1. This pseudo column to column relation will be used to generate a table-level lineage later if user need a table-level lineage model. +2. If a column in this table is used in a column to column relation while the table itself is in a table to table relation, then, this pseudo column will make it possible for a single table to includes both the column to column relation and table to table relation. + +take this SQL for example + +```sql +create view v1 as select f1 from t2; +alter table t2 rename to t3; +``` + +#### column to column relation + +![image.png](https://images.gitee.com/uploads/images/2021/0706/181125_12027549_8136809.png) + +As you can see, Table `t2` involved in the column to column relation generated by the create view statement, It also involved in a table to table relation generated by the alter table statement. A single table `t2` in the above digram show that it includes both the column to column relation and the table to table reation. + +#### table-level lineage + +With the table to table relation generated in the complete data lineage model, we can later use it to generate a table-level lineage like this: + +![image.png](https://images.gitee.com/uploads/images/2021/0707/121052_0d4d4d68_8136809.png) diff --git a/doc/the table-level lineage.md b/doc/the table-level lineage.md new file mode 100644 index 0000000..8cbe6a7 --- /dev/null +++ b/doc/the table-level lineage.md @@ -0,0 +1,87 @@ +How to get table-level model from the complete lineage model + +The table-level lineage model is built on the data of the complete data lineage model. + +1. The table id and process id in the table-level model is the same as the one in complete lineage model. +2. The new table-level model uses table and process element from the complete lineage model and generate the new relation between the table and process. +3. Iterate target and source table in the complete lineage model, ignore all intermediate dataset such as resutlset, variable, and build relation between tables. +4. Iterate table-level realtion built in step 3 and according to the processId property in the table element, create the new relation by inserting the process between 2 tables. + +```sql +create view v1 as select f1 from t2; +alter table t2 rename to t3; +``` + +### The complete data lineage + +![image.png](https://images.gitee.com/uploads/images/2021/0707/122910_64fb520d_8136809.png) + +```xml + + + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + +``` + +### The table-level lineage + +![image.png](https://images.gitee.com/uploads/images/2021/0707/122930_c6d8cb2a_8136809.png) + +```xml + + + + + +
+ + + + + + + + + + + + + + + + + + +``` + +## The export format of the table-level model + +## SQLFlow UI + +![image.png](https://images.gitee.com/uploads/images/2021/0707/145217_cf9a983c_8136809.png) diff --git a/doc/widget/gudu-sqlflow-settings.png b/doc/widget/gudu-sqlflow-settings.png new file mode 100644 index 0000000..8dbf82e Binary files /dev/null and b/doc/widget/gudu-sqlflow-settings.png differ diff --git a/doc/widget/readme.md b/doc/widget/readme.md new file mode 100644 index 0000000..536776d --- /dev/null +++ b/doc/widget/readme.md @@ -0,0 +1,414 @@ +# SQLFlow widget +The SQLFlow widget is a Javascript library that enables instantaneous data lineage visualisation on your website. + +The SQLFlow widet must work together with the Gudu SQLFlow backend +in order to visualize the data lineage and provides an actionable diagram. + +### 1. Online demo + +The SQLFlow widget is shipped together with [the SQLFlow On-Premise version](https://www.gudusoft.com/sqlflow-on-premise-version/). +No online demo is available currently. + +Once the SQLFlow widget is installed on your server, you can access the SQLFlow widget with the url like: +https://127.0.0.1/widget + + +### 2. Get started + +#### Files + +``` +├── index.html +├── jquery.min.js +├── sqlflow.widget.2.4.9.css +└── sqlflow.widget.2.4.9.js +└── 1\ +└── 2\ +└── 3\ +└── ... +``` + +>Please note that the version number in the file will changed constantly. + + +Add `sqlflow.widget.2.4.9.js` in index.html, during the execution of the JS, a new iframe will be created, and the css from js will be embedded into the iframe, +no additional css is needed. + +jquery is optional, and is inlcuded for the demostration only. + +```html + + + + + widget + + + + + + +
+ + +``` + +Insert the following code in index.js: + +```js +$(async () => { + // get a instance of SQLFlow + const sqlflow = await SQLFlow.init({ + container: document.getElementById('sqlflow'), + width: 1000, + height: 315, + apiPrefix: 'http://127.0.0.1/api', + }); + + // set dbvendor property + sqlflow.vendor.set('oracle'); + + // set sql text property + sqlflow.sqltext.set(`CREATE VIEW vsal + AS + SELECT a.deptno "Department", + a.num_emp / b.total_count "Employees", + a.sal_sum / b.total_sal "Salary" + FROM (SELECT deptno, + Count() num_emp, + SUM(sal) sal_sum + FROM scott.emp + WHERE city = 'NYC' + GROUP BY deptno) a, + (SELECT Count() total_count, + SUM(sal) total_sal + FROM scott.emp + WHERE city = 'NYC') b + ;`); + + sqlflow.visualize(); +}); +``` + + +### 3. Parameter + +| name | detail | type | optional | +| --------- | ------------------------------------------------------------------ | ---------------- | ---- | +| container | the html element where sqlflow is attached | HTMLElement | no | +| apiPrefix | the url of sqlflow backend | string | no | +| width | width of container, both percent and fix length can be used like "100%", or 800px | string \| number | no | +| height | height of container, both percent and fix length can be used like "100%", or 800px | string \| number | no | + +### 4. demos + +#### 4.1 visualize sql text +Visualize the data lineage after analyzing the input SQL query. + +- input: SQL text +- output: data lineage diagram + +All necessary files are under this directory. +``` +└── 1\ +``` + + +#### 4.2 visualize a job + +Visualize the data lineage in a [SQLFlow Job](https://docs.gudusoft.com/introduction/getting-started/different-modes-in-gudu-sqlflow). +The SQLFlow job must be already created. + +- input: a SQLFlow job id, or leave it empty to view the latest job +- output: data lineage diagram + + +All necessary files are under this directory. +``` +└── 2\ +``` + +```js +$(async () => { + const sqlflow = await SQLFlow.init({ + container: document.getElementById('demo-2'), + width: 1000, + height: 800, + apiPrefix: 'http://101.43.8.206/api', + }); + + // view job detail by job id, or leave it empty to view the latest job + await sqlflow.job.lineage.viewDetailById('b38273ec356d457bb98c6b3159c53be3'); +}); +``` + +#### 4.3 visualize a job with hardcoded parameters +Visualize the data lineage of a specified table or column in a SQLFlow job. + +- input: a SQLFlow job id, or leave it empty to view the latest job +- input: database, schema, table, column. + * If the column is omitted, return the data lineage for the specified table. + * if the table and column are ommited, return the data lineage for the specified schema. + * if the schema, table and column are ommited, return the data lineage for the specified database. +- output: data lineage diagram + + +All necessary files are under this directory. +``` +└── 3\ +``` + +```js +$(async () => { + const sqlflow = await SQLFlow.init({ + container: document.getElementById('sqlflow'), + width: 1000, + height: 700, + apiPrefix: 'http://101.43.8.206/api', + }); + + // view job detail by job id, or leave it empty to view the latest job + await sqlflow.job.lineage.viewDetailById('b38273ec356d457bb98c6b3159c53be3'); + + sqlflow.job.lineage.selectGraph({ + database: 'DWDB', // + schema: 'DBO', + table: '#TMP', + column: 'NUMBER_OFOBJECTS', + }); +}); +``` + + +#### 4.4 visualize a job, accept various parameters in UI +Visualize the data lineage of a specified table or column in a SQLFlow job. + +- input: a SQLFlow job id, or leave it empty to view the latest job +- input: database, schema, table, column. + * If the column is omitted, return the data lineage for the specified table. + * if the table and column are ommited, return the data lineage for the specified schema. + * if the schema, table and column are ommited, return the data lineage for the specified database. +- output: data lineage diagram + + +All necessary files are under this directory. +``` +└── 4\ +``` + +#### 4.5 set data lineage options of SQL query +Using the setting to control the output of data lineage of a SQL query. + +![gudu sqoflow settings](gudu-sqlflow-settings.png) + +All necessary files are under this directory. +``` +└── 5\ +``` + +#### 4.6 visualize a json object embedded in html page +A json object that includes the lineage data is embedded in the html page, +SQLFlow will visualize this json object and show the actionable diagram. + +Since all layout data is included in the json file, the SQLFlow widget +will draw the diagram and needn't to connect to the SQLFlow backend. + +So, this SQLFlow widget can work without the installation of the Gudu SQLFlow. + +All necessary files are under this directory. +``` +└── 6\ +``` + +#### 4.7 visualize data lineage in a separate json file +Read and visualize the data lineage in a json file. +This json file should be accessable in the same server as the SQLFlow widget. + +Since all layout data is included in the json file, the SQLFlow widget +will draw the diagram and needn't to connect to the SQLFlow backend. + +So, this SQLFlow widget can work without the installation of the Gudu SQLFlow. + +All necessary files are under this directory. +``` +└── 7\ +``` + + + +#### 4.8 How to get error message +Show how to get error message after processing input SQL qurey. + +All necessary files are under this directory. +``` +└── 8\ +``` + + +#### 4.9 Event: add an event listener on field(column) click +Add an event listener on field(column) click, so you can get detailed +information about the field(column) that been clicked. + +All necessary files are under this directory. +``` +└── 9\ +``` + +#### 4.10,11 + +Those features are disabled currently. + +All necessary files are under this directory. +``` +└── 10,11\ +``` + +#### 4.12 Access data lineage from url dierctly + +User can access the data lineage through a url directly by specify the data lineage type, table and column. + +>All data lineage comes from the default job at the Gudu SQLFlow backend. +If no default job is set, lineage data will be retrieved from the latest job. + +``` +http://127.0.0.1/widget/12/?type=upstream&table=dbo.emp +http://127.0.0.1/widget/12/?type=upstream&table=dbo.emp&column=salary +``` + +- input + * type: upstream or downstream + * table + * column: if column is omitted, return the lineage for table. + +the table and column name in the url is case insensitive. + +All necessary files are under this directory. +``` +└── 12\ +``` + +#### 4.13 Visualize a csv file that includes lineage data + +The format of the csv +```csv +source_db,source_schema,source_table,source_column,target_db,target_schema,target_table,target_column,relation_type,effect_type +D1E9IQ1AE488E4,DBT_TESTS,STG_PAYMENTS,AMOUNT,DEFAULT,DEFAULT,RS-5,COUPON_AMOUNT,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_ORDERS,ORDER_ID,DEFAULT,DEFAULT,RS-5,ORDER_ID,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_ORDERS,STATUS,DEFAULT,DEFAULT,RS-5,STATUS,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_PAYMENTS,AMOUNT,DEFAULT,DEFAULT,RS-5,CREDIT_CARD_AMOUNT,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_PAYMENTS,AMOUNT,DEFAULT,DEFAULT,RS-5,GIFT_CARD_AMOUNT,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_ORDERS,ORDER_DATE,DEFAULT,DEFAULT,RS-5,ORDER_DATE,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_PAYMENTS,AMOUNT,DEFAULT,DEFAULT,RS-5,AMOUNT,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_ORDERS,CUSTOMER_ID,DEFAULT,DEFAULT,RS-5,CUSTOMER_ID,fdd,select +D1E9IQ1AE488E4,DBT_TESTS,STG_PAYMENTS,AMOUNT,DEFAULT,DEFAULT,RS-5,BANK_TRANSFER_AMOUNT,fdd,select +``` + +All necessary files are under this directory. +``` +└── 13\ +``` + +#### 4.14 Visualize the lineage data using Vue +The SQLFlow provides a Vue library to support Vue framework. + +``` +└── 14\ +``` + + +#### 4.15 Event: add an event listener on table click +Add an event listener on table click, so you can get detailed +information about the table that been clicked. + +``` +└── 15\ +``` + + +### 5. SQLFlow widget api reference + +#### 5.1 vendor.set(value: Vendor) + +set the type of database vendor, Vendor is an enum type. + +```ts +type Vendor = + | 'athena' + | 'azuresql' + | 'bigquery' + | 'couchbase' + | 'db2' + | 'greenplum' + | 'hana' + | 'hive' + | 'impala' + | 'informix' + | 'mdx' + | 'mysql' + | 'netezza' + | 'openedge' + | 'oracle' + | 'postgresql' + | 'presto' + | 'redshift' + | 'snowflake' + | 'sparksql' + | 'mssql' + | 'sybase' + | 'teradata' + | 'vertica'; +``` + +#### 5.2 vendor.get(): Vendor + +return the current database vendor. + +#### 5.3 sqltext.set(value: string): void + +set the input sql query that need to be processed. + +#### 5.4 sqltext.get(): string + +Return the current SQL query text. + +#### 5.5 visualize(): Promise\ + +Show the diagram related to current input SQL. + +#### 5.6 job.lineage.viewDetailById(jobId?: string): Promise\ + +Show the diagram of a specific job. if the job id is omitted, +show the diagram of the top job, if no top job is found, show the diagram of latest job. + +#### 5.7 job.lineage.selectGraph(options: { database?: string; schema?: string; table?: string; column?: string;}): Promise\ + +Show the lineage diagram of a specific table/column. `job.lineage.viewDetailById` must be called first before calling this function, +and this function can be called multiple times to return diagram for different table/columns. + + +#### 5.8 addEventListener(event: Event, callback: Callback): void + +Add an event listner. + +Callback：(data) => any + +##### 5.8.1 Event：'onFieldClick' + +This event fired when the column in the diagram is clicked. + +data： + +| Attribute | Type | Comment | +| -------- | ------------------- | ---- | +| database | string \| undefined | name | +| schema | string \| undefined | name | +| table | string \| undefined | name | +| column | string | name | + +#### 5.9 removeEventListener(event: Event, callback: Callback): void + +Remove a event listner. + +#### 5.10 removeAllEventListener(): void + +Remove all event listners. diff --git a/github/HEAD b/github/HEAD new file mode 100644 index 0000000..cb089cd --- /dev/null +++ b/github/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/github/config b/github/config new file mode 100644 index 0000000..a5900b6 --- /dev/null +++ b/github/config @@ -0,0 +1,13 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true +[remote "origin"] + url = https://github.com/sqlparser/sqlflow_public.git + fetch = +refs/heads/*:refs/remotes/origin/* +[branch "master"] + remote = origin + merge = refs/heads/master diff --git a/github/description b/github/description new file mode 100644 index 0000000..498b267 --- /dev/null +++ b/github/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/github/hooks/applypatch-msg.sample b/github/hooks/applypatch-msg.sample new file mode 100755 index 0000000..a5d7b84 --- /dev/null +++ b/github/hooks/applypatch-msg.sample @@ -0,0 +1,15 @@ +#!/bin/sh +# +# An example hook script to check the commit log message taken by +# applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. The hook is +# allowed to edit the commit message file. +# +# To enable this hook, rename this file to "applypatch-msg". + +. git-sh-setup +commitmsg="$(git rev-parse --git-path hooks/commit-msg)" +test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"} +: diff --git a/github/hooks/commit-msg.sample b/github/hooks/commit-msg.sample new file mode 100755 index 0000000..b58d118 --- /dev/null +++ b/github/hooks/commit-msg.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to check the commit log message. +# Called by "git commit" with one argument, the name of the file +# that has the commit message. The hook should exit with non-zero +# status after issuing an appropriate message if it wants to stop the +# commit. The hook is allowed to edit the commit message file. +# +# To enable this hook, rename this file to "commit-msg". + +# Uncomment the below to add a Signed-off-by line to the message. +# Doing this in a hook is a bad idea in general, but the prepare-commit-msg +# hook is more suited to it. +# +# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1" + +# This example catches duplicate Signed-off-by lines. + +test "" = "$(grep '^Signed-off-by: ' "$1" | + sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { + echo >&2 Duplicate Signed-off-by lines. + exit 1 +} diff --git a/github/hooks/fsmonitor-watchman.sample b/github/hooks/fsmonitor-watchman.sample new file mode 100755 index 0000000..23e856f --- /dev/null +++ b/github/hooks/fsmonitor-watchman.sample @@ -0,0 +1,174 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 2) and last update token +# formatted as a string and outputs to stdout a new update token and +# all files that have been modified since the update token. Paths must +# be relative to the root of the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $last_update_token) = @ARGV; + +# Uncomment for debugging +# print STDERR "$0 $version $last_update_token\n"; + +# Check the hook interface version +if ($version ne 2) { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree = get_working_dir(); + +my $retry = 1; + +my $json_pkg; +eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; +} or do { + require JSON::PP; + $json_pkg = "JSON::PP"; +}; + +launch_watchman(); + +sub launch_watchman { + my $o = watchman_query(); + if (is_work_tree_watched($o)) { + output_result($o->{clock}, @{$o->{files}}); + } +} + +sub output_result { + my ($clockid, @files) = @_; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # binmode $fh, ":utf8"; + # print $fh "$clockid\n@files\n"; + # close $fh; + + binmode STDOUT, ":utf8"; + print $clockid; + print "\0"; + local $, = "\0"; + print @files; +} + +sub watchman_clock { + my $response = qx/watchman clock "$git_work_tree"/; + die "Failed to get clock id on '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + return $json_pkg->new->utf8->decode($response); +} + +sub watchman_query { + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $last_update_token but not from the .git folder. + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. Then we're using the "expression" term to + # further constrain the results. + my $last_update_line = ""; + if (substr($last_update_token, 0, 1) eq "c") { + $last_update_token = "\"$last_update_token\""; + $last_update_line = qq[\n"since": $last_update_token,]; + } + my $query = <<" END"; + ["query", "$git_work_tree", {$last_update_line + "fields": ["name"], + "expression": ["not", ["dirname", ".git"]] + }] + END + + # Uncomment for debugging the watchman query + # open (my $fh, ">", ".git/watchman-query.json"); + # print $fh $query; + # close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + # Uncomment for debugging the watch response + # open ($fh, ">", ".git/watchman-response.json"); + # print $fh $response; + # close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + return $json_pkg->new->utf8->decode($response); +} + +sub is_work_tree_watched { + my ($output) = @_; + my $error = $output->{error}; + if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) { + $retry--; + my $response = qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + $output = $json_pkg->new->utf8->decode($response); + $error = $output->{error}; + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + # Uncomment for debugging watchman output + # open (my $fh, ">", ".git/watchman-output.out"); + # close $fh; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + my $o = watchman_clock(); + $error = $output->{error}; + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + output_result($o->{clock}, ("/")); + $last_update_token = $o->{clock}; + + eval { launch_watchman() }; + return 0; + } + + die "Watchman: $error.\n" . + "Falling back to scanning...\n" if $error; + + return 1; +} + +sub get_working_dir { + my $working_dir; + if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $working_dir = Win32::GetCwd(); + $working_dir =~ tr/\\/\//; + } else { + require Cwd; + $working_dir = Cwd::cwd(); + } + + return $working_dir; +} diff --git a/github/hooks/post-update.sample b/github/hooks/post-update.sample new file mode 100755 index 0000000..ec17ec1 --- /dev/null +++ b/github/hooks/post-update.sample @@ -0,0 +1,8 @@ +#!/bin/sh +# +# An example hook script to prepare a packed repository for use over +# dumb transports. +# +# To enable this hook, rename this file to "post-update". + +exec git update-server-info diff --git a/github/hooks/pre-applypatch.sample b/github/hooks/pre-applypatch.sample new file mode 100755 index 0000000..4142082 --- /dev/null +++ b/github/hooks/pre-applypatch.sample @@ -0,0 +1,14 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed +# by applypatch from an e-mail message. +# +# The hook should exit with non-zero status after issuing an +# appropriate message if it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-applypatch". + +. git-sh-setup +precommit="$(git rev-parse --git-path hooks/pre-commit)" +test -x "$precommit" && exec "$precommit" ${1+"$@"} +: diff --git a/github/hooks/pre-commit.sample b/github/hooks/pre-commit.sample new file mode 100755 index 0000000..e144712 --- /dev/null +++ b/github/hooks/pre-commit.sample @@ -0,0 +1,49 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git commit" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message if +# it wants to stop the commit. +# +# To enable this hook, rename this file to "pre-commit". + +if git rev-parse --verify HEAD >/dev/null 2>&1 +then + against=HEAD +else + # Initial commit: diff against an empty tree object + against=$(git hash-object -t tree /dev/null) +fi + +# If you want to allow non-ASCII filenames set this variable to true. +allownonascii=$(git config --type=bool hooks.allownonascii) + +# Redirect output to stderr. +exec 1>&2 + +# Cross platform projects tend to avoid non-ASCII filenames; prevent +# them from being added to the repository. We exploit the fact that the +# printable range starts at the space character and ends with tilde. +if [ "$allownonascii" != "true" ] && + # Note that the use of brackets around a tr range is ok here, (it's + # even required, for portability to Solaris 10's /usr/bin/tr), since + # the square bracket bytes happen to fall in the designated range. + test $(git diff --cached --name-only --diff-filter=A -z $against | + LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0 +then + cat <<\EOF +Error: Attempt to add a non-ASCII file name. + +This can cause problems if you want to work with people on other platforms. + +To be portable it is advisable to rename the file. + +If you know what you are doing you can disable this check using: + + git config hooks.allownonascii true +EOF + exit 1 +fi + +# If there are whitespace errors, print the offending file names and fail. +exec git diff-index --check --cached $against -- diff --git a/github/hooks/pre-merge-commit.sample b/github/hooks/pre-merge-commit.sample new file mode 100755 index 0000000..399eab1 --- /dev/null +++ b/github/hooks/pre-merge-commit.sample @@ -0,0 +1,13 @@ +#!/bin/sh +# +# An example hook script to verify what is about to be committed. +# Called by "git merge" with no arguments. The hook should +# exit with non-zero status after issuing an appropriate message to +# stderr if it wants to stop the merge commit. +# +# To enable this hook, rename this file to "pre-merge-commit". + +. git-sh-setup +test -x "$GIT_DIR/hooks/pre-commit" && + exec "$GIT_DIR/hooks/pre-commit" +: diff --git a/github/hooks/pre-push.sample b/github/hooks/pre-push.sample new file mode 100755 index 0000000..4ce688d --- /dev/null +++ b/github/hooks/pre-push.sample @@ -0,0 +1,53 @@ +#!/bin/sh + +# An example hook script to verify what is about to be pushed. Called by "git +# push" after it has checked the remote status, but before anything has been +# pushed. If this script exits with a non-zero status nothing will be pushed. +# +# This hook is called with the following parameters: +# +# $1 -- Name of the remote to which the push is being done +# $2 -- URL to which the push is being done +# +# If pushing without using a named remote those arguments will be equal. +# +# Information about the commits which are being pushed is supplied as lines to +# the standard input in the form: +# +# +# +# This sample shows how to prevent push of commits where the log message starts +# with "WIP" (work in progress). + +remote="$1" +url="$2" + +zero=$(git hash-object --stdin &2 "Found WIP commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 diff --git a/github/hooks/pre-rebase.sample b/github/hooks/pre-rebase.sample new file mode 100755 index 0000000..6cbef5c --- /dev/null +++ b/github/hooks/pre-rebase.sample @@ -0,0 +1,169 @@ +#!/bin/sh +# +# Copyright (c) 2006, 2008 Junio C Hamano +# +# The "pre-rebase" hook is run just before "git rebase" starts doing +# its job, and can prevent the command from running by exiting with +# non-zero status. +# +# The hook is called with the following parameters: +# +# $1 -- the upstream the series was forked from. +# $2 -- the branch being rebased (or empty when rebasing the current branch). +# +# This sample shows how to prevent topic branches that are already +# merged to 'next' branch from getting rebased, because allowing it +# would result in rebasing already published history. + +publish=next +basebranch="$1" +if test "$#" = 2 +then + topic="refs/heads/$2" +else + topic=`git symbolic-ref HEAD` || + exit 0 ;# we do not interrupt rebasing detached HEAD +fi + +case "$topic" in +refs/heads/??/*) + ;; +*) + exit 0 ;# we do not interrupt others. + ;; +esac + +# Now we are dealing with a topic branch being rebased +# on top of master. Is it OK to rebase it? + +# Does the topic really exist? +git show-ref -q "$topic" || { + echo >&2 "No such branch $topic" + exit 1 +} + +# Is topic fully merged to master? +not_in_master=`git rev-list --pretty=oneline ^master "$topic"` +if test -z "$not_in_master" +then + echo >&2 "$topic is fully merged to master; better remove it." + exit 1 ;# we could allow it, but there is no point. +fi + +# Is topic ever merged to next? If so you should not be rebasing it. +only_next_1=`git rev-list ^master "^$topic" ${publish} | sort` +only_next_2=`git rev-list ^master ${publish} | sort` +if test "$only_next_1" = "$only_next_2" +then + not_in_topic=`git rev-list "^$topic" master` + if test -z "$not_in_topic" + then + echo >&2 "$topic is already up to date with master" + exit 1 ;# we could allow it, but there is no point. + else + exit 0 + fi +else + not_in_next=`git rev-list --pretty=oneline ^${publish} "$topic"` + /usr/bin/perl -e ' + my $topic = $ARGV[0]; + my $msg = "* $topic has commits already merged to public branch:\n"; + my (%not_in_next) = map { + /^([0-9a-f]+) /; + ($1 => 1); + } split(/\n/, $ARGV[1]); + for my $elem (map { + /^([0-9a-f]+) (.*)$/; + [$1 => $2]; + } split(/\n/, $ARGV[2])) { + if (!exists $not_in_next{$elem->[0]}) { + if ($msg) { + print STDERR $msg; + undef $msg; + } + print STDERR " $elem->[1]\n"; + } + } + ' "$topic" "$not_in_next" "$not_in_master" + exit 1 +fi + +<<\DOC_END + +This sample hook safeguards topic branches that have been +published from being rewound. + +The workflow assumed here is: + + * Once a topic branch forks from "master", "master" is never + merged into it again (either directly or indirectly). + + * Once a topic branch is fully cooked and merged into "master", + it is deleted. If you need to build on top of it to correct + earlier mistakes, a new topic branch is created by forking at + the tip of the "master". This is not strictly necessary, but + it makes it easier to keep your history simple. + + * Whenever you need to test or publish your changes to topic + branches, merge them into "next" branch. + +The script, being an example, hardcodes the publish branch name +to be "next", but it is trivial to make it configurable via +$GIT_DIR/config mechanism. + +With this workflow, you would want to know: + +(1) ... if a topic branch has ever been merged to "next". Young + topic branches can have stupid mistakes you would rather + clean up before publishing, and things that have not been + merged into other branches can be easily rebased without + affecting other people. But once it is published, you would + not want to rewind it. + +(2) ... if a topic branch has been fully merged to "master". + Then you can delete it. More importantly, you should not + build on top of it -- other people may already want to + change things related to the topic as patches against your + "master", so if you need further changes, it is better to + fork the topic (perhaps with the same name) afresh from the + tip of "master". + +Let's look at this example: + + o---o---o---o---o---o---o---o---o---o "next" + / / / / + / a---a---b A / / + / / / / + / / c---c---c---c B / + / / / \ / + / / / b---b C \ / + / / / / \ / + ---o---o---o---o---o---o---o---o---o---o---o "master" + + +A, B and C are topic branches. + + * A has one fix since it was merged up to "next". + + * B has finished. It has been fully merged up to "master" and "next", + and is ready to be deleted. + + * C has not merged to "next" at all. + +We would want to allow C to be rebased, refuse A, and encourage +B to be deleted. + +To compute (1): + + git rev-list ^master ^topic next + git rev-list ^master next + + if these match, topic has not merged in next at all. + +To compute (2): + + git rev-list master..topic + + if this is empty, it is fully merged to "master". + +DOC_END diff --git a/github/hooks/pre-receive.sample b/github/hooks/pre-receive.sample new file mode 100755 index 0000000..a1fd29e --- /dev/null +++ b/github/hooks/pre-receive.sample @@ -0,0 +1,24 @@ +#!/bin/sh +# +# An example hook script to make use of push options. +# The example simply echoes all push options that start with 'echoback=' +# and rejects all pushes when the "reject" push option is used. +# +# To enable this hook, rename this file to "pre-receive". + +if test -n "$GIT_PUSH_OPTION_COUNT" +then + i=0 + while test "$i" -lt "$GIT_PUSH_OPTION_COUNT" + do + eval "value=\$GIT_PUSH_OPTION_$i" + case "$value" in + echoback=*) + echo "echo from the pre-receive-hook: ${value#*=}" >&2 + ;; + reject) + exit 1 + esac + i=$((i + 1)) + done +fi diff --git a/github/hooks/prepare-commit-msg.sample b/github/hooks/prepare-commit-msg.sample new file mode 100755 index 0000000..10fa14c --- /dev/null +++ b/github/hooks/prepare-commit-msg.sample @@ -0,0 +1,42 @@ +#!/bin/sh +# +# An example hook script to prepare the commit log message. +# Called by "git commit" with the name of the file that has the +# commit message, followed by the description of the commit +# message's source. The hook's purpose is to edit the commit +# message file. If the hook fails with a non-zero status, +# the commit is aborted. +# +# To enable this hook, rename this file to "prepare-commit-msg". + +# This hook includes three examples. The first one removes the +# "# Please enter the commit message..." help message. +# +# The second includes the output of "git diff --name-status -r" +# into the message, just before the "git status" output. It is +# commented because it doesn't cope with --amend or with squashed +# commits. +# +# The third example adds a Signed-off-by line to the message, that can +# still be edited. This is rarely a good idea. + +COMMIT_MSG_FILE=$1 +COMMIT_SOURCE=$2 +SHA1=$3 + +/usr/bin/perl -i.bak -ne 'print unless(m/^. Please enter the commit message/..m/^#$/)' "$COMMIT_MSG_FILE" + +# case "$COMMIT_SOURCE,$SHA1" in +# ,|template,) +# /usr/bin/perl -i.bak -pe ' +# print "\n" . `git diff --cached --name-status -r` +# if /^#/ && $first++ == 0' "$COMMIT_MSG_FILE" ;; +# *) ;; +# esac + +# SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p') +# git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE" +# if test -z "$COMMIT_SOURCE" +# then +# /usr/bin/perl -i.bak -pe 'print "\n" if !$first_line++' "$COMMIT_MSG_FILE" +# fi diff --git a/github/hooks/push-to-checkout.sample b/github/hooks/push-to-checkout.sample new file mode 100755 index 0000000..af5a0c0 --- /dev/null +++ b/github/hooks/push-to-checkout.sample @@ -0,0 +1,78 @@ +#!/bin/sh + +# An example hook script to update a checked-out tree on a git push. +# +# This hook is invoked by git-receive-pack(1) when it reacts to git +# push and updates reference(s) in its repository, and when the push +# tries to update the branch that is currently checked out and the +# receive.denyCurrentBranch configuration variable is set to +# updateInstead. +# +# By default, such a push is refused if the working tree and the index +# of the remote repository has any difference from the currently +# checked out commit; when both the working tree and the index match +# the current commit, they are updated to match the newly pushed tip +# of the branch. This hook is to be used to override the default +# behaviour; however the code below reimplements the default behaviour +# as a starting point for convenient modification. +# +# The hook receives the commit with which the tip of the current +# branch is going to be updated: +commit=$1 + +# It can exit with a non-zero status to refuse the push (when it does +# so, it must not modify the index or the working tree). +die () { + echo >&2 "$*" + exit 1 +} + +# Or it can make any necessary changes to the working tree and to the +# index to bring them to the desired state when the tip of the current +# branch is updated to the new commit, and exit with a zero status. +# +# For example, the hook can simply run git read-tree -u -m HEAD "$1" +# in order to emulate git fetch that is run in the reverse direction +# with git push, as the two-tree form of git read-tree -u -m is +# essentially the same as git switch or git checkout that switches +# branches while keeping the local changes in the working tree that do +# not interfere with the difference between the branches. + +# The below is a more-or-less exact translation to shell of the C code +# for the default behaviour for git's push-to-checkout hook defined in +# the push_to_deploy() function in builtin/receive-pack.c. +# +# Note that the hook will be executed from the repository directory, +# not from the working tree, so if you want to perform operations on +# the working tree, you will have to adapt your code accordingly, e.g. +# by adding "cd .." or using relative paths. + +if ! git update-index -q --ignore-submodules --refresh +then + die "Up-to-date check failed" +fi + +if ! git diff-files --quiet --ignore-submodules -- +then + die "Working directory has unstaged changes" +fi + +# This is a rough translation of: +# +# head_has_history() ? "HEAD" : EMPTY_TREE_SHA1_HEX +if git cat-file -e HEAD 2>/dev/null +then + head=HEAD +else + head=$(git hash-object -t tree --stdin &2 + echo " (if you want, you could supply GIT_DIR then run" >&2 + echo " $0 )" >&2 + exit 1 +fi + +if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# --- Config +allowunannotated=$(git config --type=bool hooks.allowunannotated) +allowdeletebranch=$(git config --type=bool hooks.allowdeletebranch) +denycreatebranch=$(git config --type=bool hooks.denycreatebranch) +allowdeletetag=$(git config --type=bool hooks.allowdeletetag) +allowmodifytag=$(git config --type=bool hooks.allowmodifytag) + +# check for no description +projectdesc=$(sed -e '1q' "$GIT_DIR/description") +case "$projectdesc" in +"Unnamed repository"* | "") + echo "*** Project description file hasn't been set" >&2 + exit 1 + ;; +esac + +# --- Check types +# if $newrev is 0000...0000, it's a commit to delete a ref. +zero=$(git hash-object --stdin &2 + echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2 + exit 1 + fi + ;; + refs/tags/*,delete) + # delete tag + if [ "$allowdeletetag" != "true" ]; then + echo "*** Deleting a tag is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/tags/*,tag) + # annotated tag + if [ "$allowmodifytag" != "true" ] && git rev-parse $refname > /dev/null 2>&1 + then + echo "*** Tag '$refname' already exists." >&2 + echo "*** Modifying a tag is not allowed in this repository." >&2 + exit 1 + fi + ;; + refs/heads/*,commit) + # branch + if [ "$oldrev" = "$zero" -a "$denycreatebranch" = "true" ]; then + echo "*** Creating a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/heads/*,delete) + # delete branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + refs/remotes/*,commit) + # tracking branch + ;; + refs/remotes/*,delete) + # delete tracking branch + if [ "$allowdeletebranch" != "true" ]; then + echo "*** Deleting a tracking branch is not allowed in this repository" >&2 + exit 1 + fi + ;; + *) + # Anything else (is there anything else?) + echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2 + exit 1 + ;; +esac + +# --- Finished +exit 0 diff --git a/github/index b/github/index new file mode 100644 index 0000000..ea92b40 Binary files /dev/null and b/github/index differ diff --git a/github/info/exclude b/github/info/exclude new file mode 100644 index 0000000..a5196d1 --- /dev/null +++ b/github/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/github/logs/HEAD b/github/logs/HEAD new file mode 100644 index 0000000..216656c --- /dev/null +++ b/github/logs/HEAD @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 d78df2afea5f60a7e4fd09c00945b007eaae50fb j 1729677148 +0800 clone: from https://github.com/sqlparser/sqlflow_public.git diff --git a/github/logs/refs/heads/master b/github/logs/refs/heads/master new file mode 100644 index 0000000..216656c --- /dev/null +++ b/github/logs/refs/heads/master @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 d78df2afea5f60a7e4fd09c00945b007eaae50fb j 1729677148 +0800 clone: from https://github.com/sqlparser/sqlflow_public.git diff --git a/github/logs/refs/remotes/origin/HEAD b/github/logs/refs/remotes/origin/HEAD new file mode 100644 index 0000000..216656c --- /dev/null +++ b/github/logs/refs/remotes/origin/HEAD @@ -0,0 +1 @@ +0000000000000000000000000000000000000000 d78df2afea5f60a7e4fd09c00945b007eaae50fb j 1729677148 +0800 clone: from https://github.com/sqlparser/sqlflow_public.git diff --git a/github/objects/pack/pack-597b0b9933699f2f58c302c9f9c73f462ce42287.idx b/github/objects/pack/pack-597b0b9933699f2f58c302c9f9c73f462ce42287.idx new file mode 100644 index 0000000..09db3ad Binary files /dev/null and b/github/objects/pack/pack-597b0b9933699f2f58c302c9f9c73f462ce42287.idx differ diff --git a/github/objects/pack/pack-597b0b9933699f2f58c302c9f9c73f462ce42287.pack b/github/objects/pack/pack-597b0b9933699f2f58c302c9f9c73f462ce42287.pack new file mode 100644 index 0000000..1d4f8f6 Binary files /dev/null and b/github/objects/pack/pack-597b0b9933699f2f58c302c9f9c73f462ce42287.pack differ diff --git a/github/packed-refs b/github/packed-refs new file mode 100644 index 0000000..798a21b --- /dev/null +++ b/github/packed-refs @@ -0,0 +1,29 @@ +# pack-refs with: peeled fully-peeled sorted +fde56f44dd405ffde863f12e4ffda15cd56982c8 refs/remotes/origin/I7A9SH +cbeac25fd6a54678db5a9eac76df8b9641474255 refs/remotes/origin/feature/shenhuan +e1414455d44a2889fbd38e9a85b2068f43f9ad57 refs/remotes/origin/lh/main +d78df2afea5f60a7e4fd09c00945b007eaae50fb refs/remotes/origin/master +f0139dbb11219bfdc3712d15a6c3cc7022857a7f refs/tags/sqlflow-ingester1.0 +f0139dbb11219bfdc3712d15a6c3cc7022857a7f refs/tags/sqlflow-ingester1.1 +f0139dbb11219bfdc3712d15a6c3cc7022857a7f refs/tags/sqlflow-ingester1.1.1 +f0139dbb11219bfdc3712d15a6c3cc7022857a7f refs/tags/sqlflow-ingester1.1.2 +3af5c677867fc015bdeb996920c8a847d36e73e5 refs/tags/sqlflow-ingester1.1.3 +8062beb87f9514d5efa447da0df3f44d8beb8977 refs/tags/sqlflow-ingester1.1.4 +780becae52c38436a7d982483e4fc8138c2094fa refs/tags/sqlflow-ingester1.1.5 +780becae52c38436a7d982483e4fc8138c2094fa refs/tags/sqlflow-ingester1.1.6 +1eaf168facb48e03dd7b498d1cf424306384c173 refs/tags/sqlflow-ingester1.1.7 +204b0a44ed8b3cf8df977674d9bd0af699a0caf0 refs/tags/sqlflow-ingester1.1.8 +204b0a44ed8b3cf8df977674d9bd0af699a0caf0 refs/tags/sqlflow-ingester1.1.9 +6773995af6cb369350e7498799e089d7dc8fa9a0 refs/tags/sqlflow-ingester1.2.0 +6773995af6cb369350e7498799e089d7dc8fa9a0 refs/tags/sqlflow-ingester1.2.1 +6773995af6cb369350e7498799e089d7dc8fa9a0 refs/tags/sqlflow-ingester1.2.2 +6773995af6cb369350e7498799e089d7dc8fa9a0 refs/tags/sqlflow-ingester1.2.3 +c73a03ca930d7af249137b95eb40c78f8d0b0a41 refs/tags/sqlflow-ingester1.2.4 +c73a03ca930d7af249137b95eb40c78f8d0b0a41 refs/tags/sqlflow-ingester1.2.5 +c73a03ca930d7af249137b95eb40c78f8d0b0a41 refs/tags/sqlflow-ingester1.2.6 +668c7adc4bf6bdd37465e84db85b818d772ab29d refs/tags/sqlflow-ingester1.2.7 +668c7adc4bf6bdd37465e84db85b818d772ab29d refs/tags/sqlflow-ingester1.2.8 +668c7adc4bf6bdd37465e84db85b818d772ab29d refs/tags/sqlflow-ingester1.2.9 +668c7adc4bf6bdd37465e84db85b818d772ab29d refs/tags/sqlflow-ingester1.3.0 +668c7adc4bf6bdd37465e84db85b818d772ab29d refs/tags/sqlflow-ingester1.3.1 +668c7adc4bf6bdd37465e84db85b818d772ab29d refs/tags/sqlflow-ingester1.3.2 diff --git a/github/refs/heads/master b/github/refs/heads/master new file mode 100644 index 0000000..4beddf6 --- /dev/null +++ b/github/refs/heads/master @@ -0,0 +1 @@ +d78df2afea5f60a7e4fd09c00945b007eaae50fb diff --git a/github/refs/remotes/origin/HEAD b/github/refs/remotes/origin/HEAD new file mode 100644 index 0000000..6efe28f --- /dev/null +++ b/github/refs/remotes/origin/HEAD @@ -0,0 +1 @@ +ref: refs/remotes/origin/master diff --git a/grabit/github-bitbucket/bitbucket-config-template.json b/grabit/github-bitbucket/bitbucket-config-template.json new file mode 100644 index 0000000..5d9c158 --- /dev/null +++ b/grabit/github-bitbucket/bitbucket-config-template.json @@ -0,0 +1,18 @@ +{ + "SQLScriptSource":"gitserver", + "lineageReturnFormat":"json", + "databaseType":"sparksql", + "gitserver":{ + "url":"https://bitbucket.org/username/repo", + "username":"your user name", + "password":"your password", + "sshkeyPath":"" + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your user id", + "userSecret":"your secret code" + }, + "enableGetMetadataInJSONFromDatabase":0 +} diff --git a/grabit/github-bitbucket/github-snowflake.md b/grabit/github-bitbucket/github-snowflake.md new file mode 100644 index 0000000..e4e847f --- /dev/null +++ b/grabit/github-bitbucket/github-snowflake.md @@ -0,0 +1,95 @@ +## Automated data lineage from SQL scripts in Github/bitbucket + +Grabit able to fetch SQL scripts from the GitHub/bitbucket repo, +and automated data lineage from those SQL scripts by sending it +to the SQLFlow server. + +In this article, we will show you how to fetch snowflake SQL scripts +in [a github repo](https://github.com/sqlparser/snowflake-data-lineage) and +sent it to the SQLFlow for analyzing to get the data lineage visually. + +### Software used in this solution +- [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- [Grabit tool](https://www.gudusoft.com/grabit/) for SQLFlow. It's free. + +You may [request a 30 days SQLFlow on-premise version](https://www.gudusoft.com/submit-a-ticket/) +by filling out a form with the subject: request a 30 days SQLFlow on-premise version. + +Our team will contact you in 1-3 working days after receiving the message. + + +### Prerequisites +- A Linux/mac/windows server with at least 8GB memory (ubuntu 20.04 is recommended). +- Java 8 +- Nginx web server. +- Port needs to be opened. (80, 8761,8081,8083. Only 80 port need to be opened if you set up the Nginx reverse proxy as mentioned in [this document](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow.md)) + +### Install SQLFlow on-premise version +- [Guilde for install on linux](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow.md) +- [Guilde for install on window](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow_on_windows.md) +- [Guilde for install on mac](https://github.com/sqlparser/sqlflow_public/blob/master/install_sqlflow_on_mac.md) + +### Install grabit tool +After [download grabit tool](https://www.gudusoft.com/grabit/), please [check this article](https://github.com/sqlparser/sqlflow_public/tree/master/grabit) +to see how to setup the grabit tool. + + +### Install git +- **ubuntu:** +``` +sudo apt-get install git +``` +- **centos:** +``` +sudo yum install git +``` +- **mac:** +``` +brew install git +``` +- **windows:** +``` +1.Go to the Git official website to download, website address: https://git-scm.com/downloads +2.Run the Git installation file and click Next to finish the installation +``` +After the installation is completed, run **git --version** to check it is installed successfully. + +- **Generate the ssh public and private key**： +``` +ssh-keygen -o +``` + + +### Set up grabit configuration file + +`optionType=2` tells the grabit tool the SQL scripts are located in a github repo. + +``` +{ + "optionType":2, + "resultType":1, + "databaseType":"snowflake", + "SQLFlowServer":{ + "server":"http://111.229.12.71", + "serverPort":"8081", + "userId":"gudu|0123456789", + "userSecret":"" + }, + "githubRepo":{ + "url":"https://github.com/sqlparser/snowflake-data-lineage", + "username":"", + "password":"", + "sshkeyPath":"" + } +} +``` + +For other detailed information, please [check here](https://github.com/sqlparser/sqlflow_public/tree/master/grabit#6-githubrepo--bitbucketrepo) + +Now, you can get the full data lineage like this: +![snowflake data lineage](./snowflake-data-lineage.png "snowflake data lineage") + + +### Know-How +![sqlflow-automated-data-lineage](../../images/sqlflow_automated_data_lineage.png "SQLFlow automated data lineage") + diff --git a/grabit/github-bitbucket/readme.md b/grabit/github-bitbucket/readme.md new file mode 100644 index 0000000..5f12bbc --- /dev/null +++ b/grabit/github-bitbucket/readme.md @@ -0,0 +1,103 @@ +Grab files in a Github/Bitbucket repository and send them to the SQLFlow +for analyzing and return the data lineage. + +### 1. Prerequisites + +#### Install Java + +- Java 8 or higher version must be installed and configured correctly. + +setup the PATH like this: (Please change the JAVA_HOME according to your environment) + +```bash +export JAVA_HOME=/usr/lib/jvm/default-java +export PATH=$JAVA_HOME/bin:$PATH +``` + +After install Java, make sure this command executed successfully: +```bash +java -version + +java version "1.8.0_281" +Java(TM) SE Runtime Environment (build 1.8.0_281-b09) +Java HotSpot(TM) 64-Bit Server VM (build 25.281-b09, mixed mode) +``` + +#### Install git + +- **ubuntu:** + +``` +sudo apt-get install git +``` + +- **centos:** + +``` +sudo yum install git +``` + +- **mac:** + +``` +brew install git +``` + +- **windows:** + +``` +1.Go to the Git official website to download, website address: https://git-scm.com/downloads +2.Run the Git installation file and click Next to finish the installation +``` + +After the installation is completed, run **git --version** to check it is installed successfully. + +```bash +git --version + +git version 2.30.1 (Apple Git-130) +``` + +### 2. Set up grabit configuration file + +Create a config file with name such as: `gitServer.conf.json` and put it under the same directory +where start.sh or start.bat file of the grabit tool is located. + +```json +{ + "SQLScriptSource":"gitserver", + "lineageReturnFormat":"json", + "databaseType":"sparksql", + "gitserver":{ + "url":"https://bitbucket.org/username/repo", + "username":"your user name", + "password":"your password", + "sshkeyPath":"" + }, + "SQLFlowServer":{ + "server":"https://api.gudusoft.com", + "serverPort":"", + "userId":"your user id", + "userSecret":"your secret code" + }, + "enableGetMetadataInJSONFromDatabase":0 +} +``` + +Please check this document (https://github.com/sqlparser/sqlflow_public/blob/master/sqlflow-userid-secret.md) if you don't know your userId and userSecret of the SQLFlow Cloud. + +### 3. Run grabit in command line + +Linux or Mac: +```bash +./start.sh /f gitServer.conf.json +``` + +Windows: +```bash +start.bat /f gitServer.conf.json +``` + +### 4. Check data lineage on SQLFlow Cloud +![image](https://user-images.githubusercontent.com/1305435/126964571-8a418d3c-1a66-4218-9173-547020a42a18.png) + diff --git a/grabit/github-bitbucket/snowflake-data-lineage.png b/grabit/github-bitbucket/snowflake-data-lineage.png new file mode 100644 index 0000000..247d2f7 Binary files /dev/null and b/grabit/github-bitbucket/snowflake-data-lineage.png differ diff --git a/grabit/grabit-overview.png b/grabit/grabit-overview.png new file mode 100644 index 0000000..8a19a01 Binary files /dev/null and b/grabit/grabit-overview.png differ diff --git a/grabit/readme.md b/grabit/readme.md new file mode 100644 index 0000000..68cca39 --- /dev/null +++ b/grabit/readme.md @@ -0,0 +1,1174 @@ +- [What is a grabit](#what-is-a-grabit) +- [Prerequisites](#prerequisites) +- [Install](#install) +- [Running the grabit tool](#running-the-grabit-tool) + * [GUI mode](#gui-mode) + * [Command line mode](#command-line-mode) + + [Configure file](#configure-file) + + [Running in command line](#running-in-command-line) + + [Run the grabit at a scheduled time](#run-the-grabit-at-a-scheduled-time) + * [Grabit log](#grabit-log) + + [Common Log Description](#common-log-description) +- [Name of directories and files](#name-of-directories-and-files) + * [lineageReturnOutputFile](#lineagereturnoutputfile) +- [Configuration](#configuration) + * [1. SQLFlow Server](#1-sqlflow-server) + + [server](#server) + + [serverPort](#serverport) + + [userId userSecret](#userid-usersecret) + * [2. SQLScriptSource](#2-sqlscriptsource) + + [2.1. enableGetMetadataInJSONFromDatabase](#21-enablegetmetadatainjsonfromdatabase) + * [3. lineageReturnFormat](#3-lineagereturnformat) + * [4. databaseType](#4-databasetype) + * [5. databaseServer](#5-databaseserver) + + [hostname](#hostname) + + [port](#port) + + [username](#username) + + [password](#password) + + [privateKeyFile](#privatekeyfile) + + [privateKeyFilePwd](#privatekeyfilepwd) + + [database](#database) + + [extractedDbsSchemas](#extracteddbsschemas) + + [excludedDbsSchemas](#excludeddbsschemas) + + [extractedStoredProcedures](#extractedstoredprocedures) + + [extractedViews](#extractedviews) + + [enableQueryHistory](#enablequeryhistory) + + [queryHistoryBlockOfTimeInMinutes](#queryhistoryblockoftimeinminutes) + + [queryHistorySqlType](#queryhistorysqltype) + + [excludedHistoryDbsSchemas](#excludedhistorydbsschemas) + + [duplicateQueryHistory](#duplicatequeryhistory) + + [snowflakeDefaultRole](#snowflakedefaultrole) + + [metaStore](#metastore) + + [custom ddl export sql](#custom-ddl-export-sql) + * [6. gitServer](#6-gitserver) + + [url](#url) + + [username](#username-1) + + [password](#password-1) + + [sshKeyPath](#sshkeypath) + * [7. SQLInSingleFile](#7-sqlinsinglefile) + * [8. SQLInDirectory](#8-sqlindirectory) + * [9. isUploadNeo4j](#9-isuploadneo4j) + * [10. neo4jConnection](#10-neo4jconnection) + * [11. isUploadAtlas](#11-isuploadatlas) + * [12. atlasServer](#12-atlasserver) + * [13. donotConnectToSQLFlowServer](#13-donotConnectToSQLFlowServer) + * [14. jobType](#14-jobType) +- [Process SQL queries in a database table](#process-sql-queries-in-a-database-table) + * [sqlsourceTableName](#sqlsourcetablename) + + [sqlsourceColumnQuerySource](#sqlsourcecolumnquerysource) + + [sqlsourceColumnQueryName](#sqlsourcecolumnqueryname) +- [Aux features](#aux-features) + * [Extract queries that surrounded by the single quote from any files](#extract-queries-that-surrounded-by-the-single-quote-from-any-files) + * [Extract queries in metadata jsosn file to a new sql file](#extract-queries-in-metadata-jsosn-file-to-a-new-sql-file) + * [Export metadata in csv to sql files](#export-metadata-in-csv-to-sql-files) + * [Encrypted password](#encrypted-password) +- [FAQ](#faq) + + +## What is a grabit + +Grabit is a supporting tool for SQLFlow, which collects SQL scripts from various data sources for SQLFlow, and then +uploading them to SQLFlow for data lineage analysis of these SQL scripts. The analysis results can be viewed in the +browser. Meanwhile, the data lineage results will be fetched to the directory where Grabit is installed, and the JSON +results can be uploaded to the Neo4j database if necessary. + +![grabit overview](grabit-overview.png) + +## Prerequisites + +- [Download grabit tool](https://www.gudusoft.com/grabit/) +- Java 8 or higher version must be installed and configured correctly. +- Grabit GUI mode only supported in Oracle Java 8 or higher version. +- Grabit Command Line mode works under both OpenJDK and Oracle JDK. + +setup the PATH like this: (Please change the JAVA_HOME according to your environment) + +``` +export JAVA_HOME=/usr/lib/jvm/default-java + +export PATH=$JAVA_HOME/bin:$PATH +``` + +## Install + +```` +unzip grabit-x.x.x.zip + +cd grabit-x.x.x +```` + +- **linux & mac open permissions** + +```` +chmod 777 *.sh +```` + +## Running the grabit tool + +The grabit tool can be running in both GUI mode and the command line mode. + +### GUI mode + +GUI mode only runs under Oracle JDK, and only a subset of features supported in the GUI mode, +The command line mode is highly recommended. + +- **mac & linux** + +``` +./start.sh +``` + +- **windows** + +``` +start.bat +``` + +### Command line mode + +#### Configure file +Configure file tells the grabit tool how to collect SQL script and what's kind +of data lineage result you like to achieve. + +A set of pre-configed config files are located under `conf-template/` directory. +You can modify it to meet your own requirement. + + +#### Running in command line +Grabit is started command-line. + +- **mac & linux** + +``` +./start.sh -f + +note: + path_to_config_file: the full path to the config file + +eg: + ./start.sh -f config.txt +``` + +- **windows** + +``` +start.bat -f + +note: + path_to_config_file: the full path to the config file + +eg: + start.bat -f config.txt +``` + +#### Run the grabit at a scheduled time + +This guide shows you how to set up a cron job in Linux, with examples. + +- **use mac & linux crontab** + +``` +cron ./start_job.sh -f + +note: + path_to_config_file: config file path + lib_path: lib directory absolute path + +e.g.: + 1. sudo vi /etc/crontab + 2. add the following statement to the last line + 1 */1 * * * ubuntu /home/ubuntu/grabit-2.4.6/start_job.sh -f /home/ubuntu/grabit-2.4.6/conf-template/oracle-config-template /home/ubuntu/grabit-2.4.6/lib + + note: + 0 */1 * * *: cron expression + ubuntu: The name of the system user performing the task + /home/ubuntu/grabit-2.4.6/start_job.sh: The path of the task script + -f /home/ubuntu/grabit-2.4.6/conf-template/oracle-config-template: config file path + /home/ubuntu/grabit-2.4.6/lib: lib directory absolute path + 3.sudo service cron restart +``` + +Please check [this document](https://phoenixnap.com/kb/set-up-cron-job-linux) for more information about cron. + + +### Grabit log + +After execution, view the `logs/graibt.log` file for the detailed information. +Variable, `$LOG_FILE` represents the log file generated by the grabit during the execution. +``` +$LOG_FILE = logs/graibt.log +``` +#### Common Log Description + +- file is not a valid file. + +The file does not exist or the file address cannot be found. + +- sqlScriptSource is valid, support source are database,gitserver,singleFile,directory + +The `sqlScriptSource` parameter is incorrectly set. Data sources are only supported from databases, remote repositories, +and files and directories + +- lineageReturnFormat is valid, support types are json,csv,graphml + +Parameter `lineageReturnFormat` is incorrectly set. The data lineage result obtained can only be in JSON, CSV, and +GraphML formats + +- export metadata in json successful. the resulting metadata is as follows + +Exporting metadata from the specified database succeeded. + +- This database is not currently supported + +Parameter `databaseType` set error, at present only support access, bigquery, couchbase, dax, db2, greenplum, hana, +hive, impala, informix, mdx, +mssql,sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake,sybase,teradata,soql,vertica,azure + +- db connect failed + +The metadata fails to be exported from the specified database. If the metadata fails to be exported, check whether the +database connection information in the `dataServer` object is correct + +- export metadata in json failed + +Failed to export metadata from the specified database. Check whether the user who logs in to the database has the +permission to obtain metadata + +- metadata is empty + +Exporting metadata from specified database is empty, please contact me for processing + +- remote warehouse url cannot be empty + +The URL in the gitServer parameter cannot be empty + +- remote warehouse pull failed + +Failed to connect to the remote warehouse. Check whether the remote warehouse connection information is correct + +- connection failed，repourl is the ssh URL + +The remote repository address is incorrect. Please check whether it is a Git address + +- remote warehouse file to zip successful. path is：xx + +Pull to a local storage address from a remote repository + +- get token from sqlflow failed + +Failed to connect to SQLFlow. Check whether connection parameters of `sqlflowServer` are correct + +- submit job to sqlflow failed, please input https with url + +Failed to submit the SQLFlow task. Check whether the URL and port of the `sqlflowServer` are correct + +- submit job to sqlflow failed + +Failed to submit the SQLFLOW task. Check whether the sqlFLOW background service is started properly + +- get job to status failed + +After a job is submitted to SQLFLOW, SQLFlow fails to execute the job + +- export json result failed + +Description Failed to export Data Lineage in JSON format from SQLflow + +- export csv result failed + +Description Failed to export Data Lineage in csv format from SQLflow + +- export diagram result failed + +Description Failed to export Data Lineage in diagram format from SQLflow + +- submit job to sqlflow successful + +The job is successfully submitted to SQLFlow, and the basic information about the submitted job is displayed + +- [database: 0 table: 0 view: 0 procedure: 0 column: 0 synonym: 0] + +Statistics the amount of metadata exported from the specified database + +- the time taken to export : 0ms + +Time, in milliseconds, to export metadata from the specified database + +- download success, path: xxx + +Local storage address of Data Lineage returned after successfully submitting a job to SQLFlow + +- job id is : xxxx + +job id from sqlflow , log in to the SQLFlow website to view the newly analyzed results. In the `Job List`, you can view +the analysis results of the currently submitted tasks. + +- The number of relationships in this task is too large to export this file, please check blood relationships on SQLFlow platform. + +When the task uploaded to SQLFlow is too large and the number of rolls parsed by SQLFlow is too large, Grabit cannot obtain CSV files from it and needs to check the blood relationship of this task on SQLFlow. + + +## Name of directories and files + +- The name of submitted job is `grabit_%yyyyMMddHHmmss%`, Variable `$JOB_NAME` represents the job name. +``` +$JOB_NAME = grabit_%yyyyMMddHHmmss% +``` + +- After export metadata from the database, the metadata data is saved under the `data/job_%jobname%/metadata` directory. + Variable `$METADATA_DIR` represents the directory when the metadata exported from a database is saved. +``` +$METADATA_DIR = data/job_$JOB_NAME/metadata +``` + +Metadata file name: `metadata.json` + +- Once the job is done, the data lineage result generated by the SQLFlow is saved under the `data/job_%jobname%/result` directory. + Variable: `$LINEAGE_RESULT_DIR` represents the directory where the lineage result of a job is saved. +``` +$LINEAGE_RESULT_DIR = data/job_$JOB_NAME/result +``` + +Lineage result file name: `data-lineage-result.json` + +### lineageReturnOutputFile +The value of `$LINEAGE_RESULT_DIR` will be overwrited by the value of `lineageReturnOutputFile` which +specify the data lineage result file directly. + + +```json +"lineageReturnOutputFile":"/user/data.csv" +``` +> PLEASE NOTE THAT ALL VARIABLE NAMES THAT START WITH $ SIGN IN THIS SECTION IS USED FOR COMMUNICATION ONLY, +> THEY DO NOT BEEN USED IN THE CONFIGURE FILE! + +## Configuration + +Modify the configure file to set all parameters correctly according to your environment. + +### 1. SQLFlow Server + +This is the SQLFlow server that the grabit sends the SQL script. + +#### server + +Usually, it is the IP address of [the SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +installed on your owner servers such as `127.0.0.1` or `http://127.0.0.1` + +You may set the value to `https://api.gudusoft.com` if you like to send your SQL script +to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com) to get the data lineage result. + +#### serverPort + +The default value is `8081` if you connect to your SQLFlow on-premise server. + +However, if you setup the nginx reverse proxy in the nginx configuration file like this: + +``` + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } +``` + +Then, keep the value of `serverPort` empty and set `server` to the value like this: `http://127.0.0.1/api`. + +> Please keep this value empty if you connect to the SQLFlow Cloud Server by specifying the `https://api.gudusoft.com` +in the `server` + +#### userId userSecret + +This is the user id that is used to connect to the SQLFlow server. Always set this value to `gudu|0123456789` and +keep `userSecret` empty if you use the SQLFlow on-premise version. + +If you want to connect to [the SQLFlow Cloud Server](https://sqlflow.gudusoft.com), you +may [request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +[get the necessary userId and secret code](/sqlflow-userid-secret.md). + +Example configuration for on-premise version: + +```json +"SQLFlowServer":{ + "server": "127.0.0.1", + "serverPort": "8081", + "userId": "gudu|0123456789", + "userSecret": "" +} +``` + +Example configuration for Cloud version: + +```json +"SQLFlowServer":{ + "server": "https://api.gudusoft.com", + "serverPort": "", + "userId": "your own user id here", + "userSecret": "your own secret key here" +} +``` + +### 2. SQLScriptSource + +You may collect SQL scripts from various sources such as database, Github repo, file system. This parameter tells grabit +where the SQL scripts come from. + +Available values for this parameter: + +- database +- gitserver +- singleFile +- directory + +This configuration means the SQL script is collected from a database. + +```JSON +"SQLScriptSource":"database" +``` + +#### 2.1. enableGetMetadataInJSONFromDatabase + +If the source of the SQL scripts is not the database, we may specify a database by setting `databaseServer` parameter to +fetch metadata from the database instance to help SQLFlow get a more accurate result during the analysis. + +If `enableGetMetadataInJSONFromDatabase=1`, You must set all necessary information in `databaseServer`. + +Of course, you can `enableGetMetadataInJSONFromDatabase=0`. This means all SQL scripts will be analyzed offline without +a connection to the database. SQLFlow still works quite well to get the data lineage result by taking advantage of its +powerful SQL analysis capability. + +Sample configuration of enable fetching metadata in json from the database: + +```json +"enableGetMetadataInJSONFromDatabase":1 +``` + +### 3. lineageReturnFormat + +When you submit SQL script to the SQLFlow server, A job is created on the SQLFlow server and you can always see the +graphic data lineage result in the frontend of the SQLFlow by using the browser, + +Even better, grabit will fetch the data lineage back to the directory where the grabit is running. Those data lineage +results are stored in the `data/job_%jobname/result/` directory. + +This parameter specifies which kind of format is used to save the data lineage result. + +Available values for this parameter: + +- json, data lineage result in JSON. +- csv, data lineage result in CSV format. +- graphml, data in graphml format that can be viewed by yEd. + +This sample configuration means the output format is json. + +```json +"lineageReturnFormat":"json" +``` + +### 4. databaseType + +This parameter specifies the database dialect of the SQL scripts that the SQLFlow has analyzed. + +```txt + access,bigquery,couchbase,dax,db2,greenplum,hana,hive,impala,informix,mdx,mssql, + sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake, + sybase,teradata,soql,vertica,azure +``` + +This sample configuration means the SQL dialect is SQL Server database. + +```json +"databaseType":"sqlserver" +``` + +### 5. databaseServer + +This option is used when +- 1. Need to discover data lineage of a specific database: `SQLScriptSource=database` +- 2. Need to discover lineage from SQL in a text file, but need metadata from a database in order to get a more accurate result. +- 3. Need to discover the Hive metaStore that stored in a database. + + +#### hostname + +The IP of the database server that the grabit connects. + +#### port + +The port number of the database server that the grabit connect. + +#### username + +The database user used to login to the database. + +#### password + +The password of the database user. + +note: the passwords can be encrypted using tools [Encrypted password](#Encrypted password), using encrypted passwords +more secure. + +#### privateKeyFile + +Use a private key to connect, Only supports the `snowflake`. + +#### privateKeyFilePwd + +Generate the password for the private key, Only supports the `snowflake`. + +#### database + +The name of the database instance to which it is connected. + +For azure,greenplum,netezza,oracle,postgresql,redshift,teradata databases, it represents the database name and is +required, For other databases, it is optional. + +` +note: +If this parameter is specified and the database to which it is connected is Azure, Greenplum, PostgreSQL, or Redshift, then only metadata under that library is extracted. +` + +#### extractedDbsSchemas + +List of databases and schemas to extract, separated by commas, which are to be provided in the format database/schema; +Or blank to extract all databases. +`database1/schema1,database2/schema2,database3` or `database1.schema1,database2.schema2,database3` +When parameter `database` is filled in, this parameter is considered a schema. And support wildcard characters such +as `database1/*`,`*/schema`,`*/*`. + +When the connected databases are `Oracle` and `Teradata`, this parameter is set the schemas, for example: + +````json +extractedDbsSchemas: "HR,SH" +```` + +When the connected databases are `Mysql` , `Sqlserver`, `Postgresql`, `Snowflake`, `Greenplum`, `Redshift`, `Netezza` +, `Azure`, this parameter is set database/schema, for example: + +````json +extractedDbsSchemas: "MY/ADMIN" +```` + +#### excludedDbsSchemas + +This parameters works under the resultset filtered by `extractedDbsSchemas`. List of databases and schemas to exclude +from extraction, separated by commas +`database1/schema1,database2` or `database1.schema1,database2` +When parameter `database` is filled in, this parameter is considered a schema. And support wildcard characters such +as `database1/*`,`*/schema`,`*/*`. + +When the connected databases are `Oracle` and `Teradata`, this parameter is set the schemas, for example: + +````json +excludedDbsSchemas: "HR" +```` + +When the connected databases are `Mysql` , `Sqlserver`, `Postgresql`, `Snowflake`, `Greenplum`, `Redshift`, `Netezza` +, `Azure`, this parameter is set database/schema, for example: + +````json +excludedDbsSchemas: "MY/*" +```` + +#### extractedStoredProcedures + +A list of stored procedures under the specified database and schema to extract, separated by commas, which are to be +provided in the format database.schema.procedureName or schema.procedureName; Or blank to extract all databases, support +expression. +`database1.schema1.procedureName1,database2.schema2.procedureName2,database3.schema3,database4` +or `database1/schema1/procedureName1,database2/schema2` + +for example: + +````json +extractedStoredProcedures: "database.scott.vEmp*" +```` + +or + +````json +extractedStoredProcedures: "database.scott" +```` + +#### extractedViews + +A list of stored views under the specified database and schema to extract, separated by commas, which are to be provided +in the format database.schema.viewName or schema.viewName. Or blank to extract all databases, support expression. +`database1.schema1.procedureName1,database2.schema2.procedureName2,database3.schema3,database4` +or `database1/schema1/procedureName1,database2/schema2` + +for example: + +````json +extractedViews: "database.scott.vEmp*" +```` + +or + +````json +extractedViews: "database.scott" +```` + +#### enableQueryHistory + +Fetch SQL queries from the query history if set to `true` default is false. + +#### queryHistoryBlockOfTimeInMinutes + +When `enableQueryHistory:true`, the interval at which the SQL query was extracted in the query History,default is `30` +minutes. + +#### queryHistorySqlType + +When `enableQueryHistory:true`, the DML type of SQL is extracted from the query History. When empty, all types are +extracted, and when multiple types are specified, a comma separates them, such as `SELECT,UPDATE,MERGE`. Currently only +the snowflake database supports this parameter,support types are **SHOW,SELECT,INSERT,UPDATE,DELETE,MERGE,CREATE TABLE, +CREATE VIEW, CREATE PROCEDURE, CREATE FUNCTION**. + +for example: + +````json +queryHistorySqlType: "SELECT,DELETE" +```` + +#### excludedHistoryDbsSchemas + +This parameters works under the resultset filtered by `excludedHistoryDbsSchemas`. List of databases and schemas to +exclude from extraction, separated by commas `database1.schema1,database2`. for example: + +````json +excludedHistoryDbsSchemas: "DB1.SCHEMA1,DB2.SCHEMA2" +```` + +#### duplicateQueryHistory + +When `duplicateQueryHistory:1`, duplicate History Queries are filtered, not filtered by default 0. + +#### snowflakeDefaultRole + +This value represents the role of the snowflake database. + +```` +note: You must define a role that has access to the SNOWFLAKE database,And assign WAREHOUSE permission to this role. +```` + +Assign permissions to a role, for example: + +````sql +#create +role +use role accountadmin; +grant imported +privileges on database snowflake to role sysadmin; +grant imported +privileges on database snowflake to role customrole1; +use +role customrole1; +select * +from snowflake.account_usage.databases; + +#To +do this, the Role gives the WAREHOUSE permission +select current_warehouse() + use role sysadmin GRANT ALL PRIVILEGES +ON WAREHOUSE %current_warehouse% TO ROLE customrole1; +```` + +> Please refer to:[Grant Privileges](https://github.com/sqlparser/sqlflow_public/blob/master/databases/snowflake/readme.md#a-minimum-list-of-permissions-need-to-extract-all-ddl) + +#### metaStore + +> ref #: I44EE8 + +This option means the metadata is fetched from a Hive or SparkSQL metaStore. +The metaStore uses the RDBMS such as MySQL to save the metadata. + +> Only Hive metaStore is supported in current version. + +when this option is set to `hive`, Grabit extract metadata from the metaStore, +but not from the common metadata of the database. + +Sample configuration of a SQL Server database: + +```json +"hostname":"127.0.0.1", +"port": "1433", +"username": "sa", +"password": "PASSWORD", +"database": "", +"extractedDbsSchemas":"AdventureWorksDW2019/dbo", +"excludedDbsSchemas": "", +"extractedStoredProcedures": "AdventureWorksDW2019.dbo.f_qry*", +"extractedViews": "", +"enableQueryHistory": false, +"queryHistoryBlockOfTimeInMinutes":30, +"snowflakeDefaultRole": "", +"queryHistorySqlType": "", +"metaStore": "hive" +``` + + +#### custom ddl export sql + +`conf.zip` file contains all ddl export sql, you can edit the sql file in the `conf.zip`, keep the same of return +fields, put the modified sql file at: conf/%database_name%/%query_type%.sql + +for example, when you edit the conf/mssql/query.sql, please copy it to conf/mssql/query.sql, the grabit will load your +modified sql file as ddl export sql. + +Please check [conf.zip](https://github.com/sqlparser/gsp_demo_java/tree/master/src/main/java/demos/dlineage/conf.zip) +download. + + +### 6. gitServer + +When `SQLScriptSource=gitserver`, grabit will fetch SQL files from a specified github or bitbucket repo, the SQL script +files are stored in `data/job_%jobname%/metadata/` before submitting to the SQLFlow server. + +Both sshkey and account password authentication methods are supported. + +#### url + +Pull the repository address of the SQL script from GitHub or BitBucket. + +` +note: If sshkey authentication is used, you must enter the SSH address. +` + +#### username + +Pull the user name to which the SQL script is connected from GitHub or BitBucket. + +#### password + +Pull the personal token to which the SQL script is connected from GitHub or BitBucket. + +#### sshKeyPath + +The full path to the SSH private key file. + +Sample configuration of the GitHub or BitBucket public repository servers : + +```json +"url":"your public repository address here", +"username": "", +"password": "", +"sshKeyPath": "" +``` + +Sample configuration of the GitHub or BitBucket private repository servers: + +```json +"url":"your private library address here", +"username": "your private repository username here", +"password": "your private repository personal token here", +"sshKeyPath": "" +``` + +or + +```json +"url":"your private repository ssh address here", +"username": "", +"password": "", +"sshKeyPath": "your private repository ssh key address here" +``` + +### 7. SQLInSingleFile + +When `SQLScriptSource=singleFile`, this is a single SQL file needs to be analyzed. + +- **filePath** + +The name of the SQL file with full path. + +- **csvFormat** + +Format of a CSV file. used to represent the CSV in the `Catalog, Schema, ObjectType, ObjectName, ObjectCode, Notes ` +each column is the number of columns in the CSV file, does not exist it is `0`, The default is `123456`. + +- **objectCodeEncloseChar** + +Specifies that the string contains SQL Code content. + +- **objectCodeEscapeChar** + +ObjectCodeEncloseChar specifies the string escape. + +- **redshiftLog** + +The default value of redshiftLog is 0, indicating whether the uploaded file is a common SQL script file. For example, if +the redshift log file needs to be parsed by SQLFlow, set the value to 1. + +- **isDumpDDL** + +The default value of isDumpDDL is 0, this value is set to 1 when the specified file is a DDL file from the database +dump. + +- **databaseName** + +The default value of databaseName is left blank. When the specified file is a DDL file from a database dump, this value +is set to the name of the database from which the DDL was retrieved. + +### 8. SQLInDirectory + +When `SQLScriptSource=directory`, SQL files under this directory including sub-directory will be analyzed. + +- **directoryPath** + +The directory includes the SQL files. + +- **redshiftLog** + +The default value of redshiftLog is 0, indicating whether the uploaded file is a common SQL script file. For example, if +the redshift log file needs to be parsed by SQLFlow, set the value to 1. + +### 9. isUploadNeo4j + +Upload the data lineage result to a Neo4j database for further processing. Available values for this parameter is 1 or +0, enable this function if the value is 1, disable it if the value is 0, the default value is 0. + +Sample configuration of a Whether to upload neo4j: + +```json +"isUploadNeo4j":1 +``` + +### 10. neo4jConnection + +If `IsuploadNeo4j` is set to '1', this parameter specifies the details of the neo4j server. + +- **url** + +The IP of the neo4j server that connects to. + +- **username** + +The user name of the neo4j server that connect to. + +- **password** + +The password of the neo4j server that connect to. + +Sample configuration of a local directory path: + +```json +"url":"127.0.0.1:7687", +"username": "your server username here", +"password": "your server password here" +``` + +### 11. isUploadAtlas + +Upload the metadata to a Atlas server for further processing. Available values for this parameter is 1 or 0, enable this +function if the value is 1, disable it if the value is 0, the default value is 0. + +Sample configuration of a Whether to upload neo4j: + +```json +"isUploadAtlas":1 +``` + +### 12. atlasServer + +If `isUploadAtlas` is set to '1', this parameter specifies the details of the atlas server. + +- **ip** + +The IP of the atlas server that connects to. + +- **port** + +The PORT of the atlas server that connects to. + +- **username** + +The user name of the atlas server that connect to. + +- **password** + +The password of the atlas server that connect to. + +Sample configuration of a local directory path: + +```json +"ip":"127.0.0.1", +"port": "21000", +"username": "your server username here", +"password": "your server password here" +``` + +### 13. donotConnectToSQLFlowServer + +If `donotConnectToSQLFlowServer` is set to 1, the metadata file is not uploaded to SQLFlow. the default is 0 + +### 14. jobType + +If `jobType` is set to "regular", tasks are submitted to SQLFlow incrementally and in batches according to reualr mode. the default is "simple" + +**eg configuration file:** + +````json +{ + "databaseServer": { + "hostname": "127.0.0.1", + "port": "1433", + "username": "sa", + "password": "PASSWORD", + "privateKeyFile": "", + "privateKeyFilePwd": "", + "database": "", + "extractedDbsSchemas": "AdventureWorksDW2019/dbo", + "excludedDbsSchemas": "", + "extractedStoredProcedures": "", + "extractedViews": "", + "enableQueryHistory": false, + "excludedHistoryDbsSchemas":"" , + "duplicateQueryHistory":0 , + "queryHistoryBlockOfTimeInMinutes": 30, + "snowflakeDefaultRole": "", + "queryHistorySqlType": "", + "metaStore": "" + }, + "gitServer": { + "url": "https://github.com/sqlparser/snowflake-data-lineage", + "username": "", + "password": "", + "sshkeyPath": "" + }, + "SQLInSingleFile": { + "filePath": "", + "csvFormat": "", + "objectCodeEncloseChar": "", + "objectCodeEscapeChar": "", + "dumpDDL": 0, + "databaseName": "" + }, + "SQLInDirectory": { + "directoryPath": "" + }, + "SQLFlowServer": { + "server": "http:127.0.0.1", + "serverPort": "8081", + "userId": "gudu|0123456789", + "userSecret": "" + }, + "neo4jConnection": { + "url": "", + "username": "", + "password": "" + }, + "atlasServer": { + "ip": "127.0.0.1", + "port": "21000", + "username": "", + "password": "" + }, + "isUploadAtlas": 0, + "SQLScriptSource": "database", + "lineageReturnFormat": "json", + "lineageReturnOutputFile": "", + "databaseType": "snowflake", + "isUploadNeo4j": 0, + "donotConnectToSQLFlowServer": 0, + "enableGetMetadataInJSONFromDatabase": 0, + "jobType": "simple" +} +```` + +## Process SQL queries in a database table +> Ref #: I43QRV + +This feature will extract SQL queries saved in a database table, +metadata of the same database will also be extracted into the same JSON file. + +Option also need to be set is `databaseServer`. + +### sqlsourceTableName + +Name of the table where SQL queries are saved. + + +table name: **query_table** + +| query_name | query_source | +| ---------- | ----------------------------------- | +| query1 | create view v1 as select f1 from t1 | +| query2 | create view v2 as select f2 from t2 | +| query3 | create view v3 as select f3 from t3 | + +If you save SQL queries in a specific table, one SQL query per row. + +Let's say: The column `query_table.query_source` stores the source code of the query. +We can use this query to fetch all SQL queries in this table: + +```sql +select query_name as queryName, query_source as querySource +from query_table +``` + +By setting the value of `sqlsourceTableName` and `sqlsourceColumnQuerySource`,`sqlsourceColumnQueryName` +grabit can fetch all SQL queries in this table and send it to the SQLFlow to analzye the lineage. + +In this example, + +``` +"sqlsourceTableName":"query_table" +"sqlsourceColumnQuerySource":"query_source" +"sqlsourceColumnQueryName":"query_name" +``` + +Please leave `sqlsourceTableName` empty if you don't fetch SQL queries from a specific table. + +#### sqlsourceColumnQuerySource + +In the above sample: + +``` +"sqlsourceColumnQuerySource":"query_source" +``` + +#### sqlsourceColumnQueryName + +``` +"sqlsourceColumnQueryName":"query_name" +``` + +This parameter is optional, you don't need to speicify a query name column if it doesn't exist in the table. + + +## Aux features +### Extract queries that surrounded by the single quote from any files + +Read text files from a specified directory, extract text that surrounded by the single quote character +from each file in the directory and put the text in a separate new file +if the text represents a valid SQL statement. + +> reference #: I48FCX + +- **mac & linux** + +``` +./start.sh -e --generic -t dbvendor dir_to_txt_file + +eg: + ./start.sh -e --generic -t oracle /root/oracledir +``` + +- **windows** + +``` +start.bat -e --generic -t dbvendor dir_to_txt_file + +eg: + start.bat -e --generic -t oracle /root/oracledir +``` + +### Extract queries in metadata jsosn file to a new sql file + +After exporting metadata from a database, the metadata is saved in a JSON file. +Queries such as create view are included in this JSON file. + +By using this option, grabit will extracts all queries in the JSON file, and put +each query in a separate text file. + +> Ref #: I3DTWP + +- **mac & linux** + +``` +./start.sh -e path_to_json_file [--targetDir target_dir] + +note: + path_to_config_file: the full path to the metedata json file + target_dir: the path to the generated SQL file, optional + +eg: + ./start.sh -e test.json --targetDir /root/sqlfiles +``` + +- **windows** + +``` +start.bat -e path_to_json_file [--targetDir target_dir] + +note: + path_to_config_file: the full path to the metedata json file + target_dir: the path to the generated SQL file, optional + +eg: + start.bat -e test.json --targetDir /root/sqlfiles +``` + +### Export metadata in csv to sql files + +Extract SQL queries in a CSV file and save each SQL query in a separate text file. + +Detailed descrition of the CSV file format, please [check this document](https://www.gudusoft.com/blog/2021/09/05/sqlflow-csv/) +> Ref #: I45ANU + +- **mac & linux** + +``` +./start.sh -e --csv path_to_csv_file [--csvFormat 123456] [--objectCodeEncloseChar char] [--objectCodeEscapeChar char] [--targetDir target_dir] + +note: + path_to_config_file: the full path to the metedata csv file + target_dir: the path to the generated SQL file, optional + +eg: + ./start.sh -e --csv test.csv --csvFormat 123456 --objectCodeEscapeChar " --objectCodeEncloseChar " --targetDir /root/sqlfiles +``` + +- **windows** + +``` +start.bat -e --csv path_to_csv_file [--csvFormat 123456] [--objectCodeEncloseChar char] [--objectCodeEscapeChar char] [--targetDir target_dir] + +note: + path_to_config_file: the full path to the metedata csv file + target_dir: the path to the generated SQL file, optional + +eg: + start.bat -e --csv test.csv --csvFormat 123456 --objectCodeEscapeChar " --objectCodeEncloseChar " --targetDir /root/sqlfiles +``` + +### Encrypted password + +Encrypt the database connection password. + +- **mac & linux** + +``` +./start.sh --encrypt password + +note: + password: the database connection password + +eg: + ./start.sh --encrypt 123456 +``` + +- **windows** + +``` +./start.bat --encrypt password + +note: + password: the database connection password + +eg: + ./start.bat --encrypt 123456 +``` + +## FAQ +- Q: Is it possible to upload multiple scripts and view the lineage across these scripts in a combined way? + +> A: Yes, you can add multiple scripts to a single zip file and then upload this zip file to SQLFlow by creating a job + diff --git a/images/SQL & Metadata Sources_1000_1721_trabsit.png b/images/SQL & Metadata Sources_1000_1721_trabsit.png new file mode 100644 index 0000000..25a0e55 Binary files /dev/null and b/images/SQL & Metadata Sources_1000_1721_trabsit.png differ diff --git a/images/SQL & Metadata Sources_700_988_transit.png b/images/SQL & Metadata Sources_700_988_transit.png new file mode 100644 index 0000000..b625bab Binary files /dev/null and b/images/SQL & Metadata Sources_700_988_transit.png differ diff --git a/images/SQL & Metadata Sources_700_988_white.png b/images/SQL & Metadata Sources_700_988_white.png new file mode 100644 index 0000000..b749392 Binary files /dev/null and b/images/SQL & Metadata Sources_700_988_white.png differ diff --git a/images/SQL and Metadata Sources.png b/images/SQL and Metadata Sources.png new file mode 100644 index 0000000..e45bbba Binary files /dev/null and b/images/SQL and Metadata Sources.png differ diff --git a/images/apply-premium-account-step1.png b/images/apply-premium-account-step1.png new file mode 100644 index 0000000..dce815c Binary files /dev/null and b/images/apply-premium-account-step1.png differ diff --git a/images/apply-premium-account-step2.png b/images/apply-premium-account-step2.png new file mode 100644 index 0000000..1d71c45 Binary files /dev/null and b/images/apply-premium-account-step2.png differ diff --git a/images/bigquery-create-external-table.png b/images/bigquery-create-external-table.png new file mode 100644 index 0000000..183b63b Binary files /dev/null and b/images/bigquery-create-external-table.png differ diff --git a/images/dbt-tables/create-view-column-level-data-lineage.png b/images/dbt-tables/create-view-column-level-data-lineage.png new file mode 100644 index 0000000..e58b08c Binary files /dev/null and b/images/dbt-tables/create-view-column-level-data-lineage.png differ diff --git a/images/dbt-tables/insert-column-level-data-lineage.png b/images/dbt-tables/insert-column-level-data-lineage.png new file mode 100644 index 0000000..a2d1d85 Binary files /dev/null and b/images/dbt-tables/insert-column-level-data-lineage.png differ diff --git a/images/dbt-tables/insert-table-level-data-lineage.png b/images/dbt-tables/insert-table-level-data-lineage.png new file mode 100644 index 0000000..59b77b0 Binary files /dev/null and b/images/dbt-tables/insert-table-level-data-lineage.png differ diff --git a/images/dbt-tables/merge-column-level-data-lineage.png b/images/dbt-tables/merge-column-level-data-lineage.png new file mode 100644 index 0000000..a74e8ef Binary files /dev/null and b/images/dbt-tables/merge-column-level-data-lineage.png differ diff --git a/images/gudu-sqlflow-license.png b/images/gudu-sqlflow-license.png new file mode 100644 index 0000000..9d6d88a Binary files /dev/null and b/images/gudu-sqlflow-license.png differ diff --git a/images/hive-load-data.png b/images/hive-load-data.png new file mode 100644 index 0000000..12254e4 Binary files /dev/null and b/images/hive-load-data.png differ diff --git a/images/pivot_clause.png b/images/pivot_clause.png new file mode 100644 index 0000000..cf67de9 Binary files /dev/null and b/images/pivot_clause.png differ diff --git a/images/sparksql-insert-overwrite-directory.png b/images/sparksql-insert-overwrite-directory.png new file mode 100644 index 0000000..948310c Binary files /dev/null and b/images/sparksql-insert-overwrite-directory.png differ diff --git a/images/sparksql-pivot-clause.png b/images/sparksql-pivot-clause.png new file mode 100644 index 0000000..4d976ef Binary files /dev/null and b/images/sparksql-pivot-clause.png differ diff --git a/images/sparksql-pivot.png b/images/sparksql-pivot.png new file mode 100644 index 0000000..745a8d4 Binary files /dev/null and b/images/sparksql-pivot.png differ diff --git a/images/sqlflow-connect-to-database-from-database.png b/images/sqlflow-connect-to-database-from-database.png new file mode 100644 index 0000000..b1c3768 Binary files /dev/null and b/images/sqlflow-connect-to-database-from-database.png differ diff --git a/images/sqlflow-connect-to-database-job-list.png b/images/sqlflow-connect-to-database-job-list.png new file mode 100644 index 0000000..3517811 Binary files /dev/null and b/images/sqlflow-connect-to-database-job-list.png differ diff --git a/images/sqlflow-install-502-bad-gateway.png b/images/sqlflow-install-502-bad-gateway.png new file mode 100644 index 0000000..44d265b Binary files /dev/null and b/images/sqlflow-install-502-bad-gateway.png differ diff --git a/images/sqlflow-install-customize-gsplive-port-nginx.png b/images/sqlflow-install-customize-gsplive-port-nginx.png new file mode 100644 index 0000000..68f1905 Binary files /dev/null and b/images/sqlflow-install-customize-gsplive-port-nginx.png differ diff --git a/images/sqlflow-install-customize-port-gsplive.png b/images/sqlflow-install-customize-port-gsplive.png new file mode 100644 index 0000000..e4442f5 Binary files /dev/null and b/images/sqlflow-install-customize-port-gsplive.png differ diff --git a/images/sqlflow-install-customize-web-port.png b/images/sqlflow-install-customize-web-port.png new file mode 100644 index 0000000..63b09b9 Binary files /dev/null and b/images/sqlflow-install-customize-web-port.png differ diff --git a/images/sqlflow-install-failed-to-get-license-info.png b/images/sqlflow-install-failed-to-get-license-info.png new file mode 100644 index 0000000..a849ba0 Binary files /dev/null and b/images/sqlflow-install-failed-to-get-license-info.png differ diff --git a/images/sqlflow-main-ui.png b/images/sqlflow-main-ui.png new file mode 100644 index 0000000..ca4d7c6 Binary files /dev/null and b/images/sqlflow-main-ui.png differ diff --git a/images/sqlflow-overview-grabit-big.png b/images/sqlflow-overview-grabit-big.png new file mode 100644 index 0000000..d7f74e6 Binary files /dev/null and b/images/sqlflow-overview-grabit-big.png differ diff --git a/images/sqlflow-overview-grabit.png b/images/sqlflow-overview-grabit.png new file mode 100644 index 0000000..4dd0b17 Binary files /dev/null and b/images/sqlflow-overview-grabit.png differ diff --git a/images/sqlflow-overview-phthon-big.png b/images/sqlflow-overview-phthon-big.png new file mode 100644 index 0000000..b0c1f5b Binary files /dev/null and b/images/sqlflow-overview-phthon-big.png differ diff --git a/images/sqlflow-overview-phthon.png b/images/sqlflow-overview-phthon.png new file mode 100644 index 0000000..31c5b49 Binary files /dev/null and b/images/sqlflow-overview-phthon.png differ diff --git a/images/sqlflow-search-table-column.gif b/images/sqlflow-search-table-column.gif new file mode 100644 index 0000000..ed032c0 Binary files /dev/null and b/images/sqlflow-search-table-column.gif differ diff --git a/images/sqlflow-snowflake-data-lineage-aws.PNG b/images/sqlflow-snowflake-data-lineage-aws.PNG new file mode 100644 index 0000000..87b27d8 Binary files /dev/null and b/images/sqlflow-snowflake-data-lineage-aws.PNG differ diff --git a/images/sqlflow-team-manager.png b/images/sqlflow-team-manager.png new file mode 100644 index 0000000..e19d38b Binary files /dev/null and b/images/sqlflow-team-manager.png differ diff --git a/images/sqlflow-team-menu.png b/images/sqlflow-team-menu.png new file mode 100644 index 0000000..6928cb1 Binary files /dev/null and b/images/sqlflow-team-menu.png differ diff --git a/images/sqlflow-tutorial-search-table-visualize.gif b/images/sqlflow-tutorial-search-table-visualize.gif new file mode 100644 index 0000000..1096b69 Binary files /dev/null and b/images/sqlflow-tutorial-search-table-visualize.gif differ diff --git a/images/sqlflow_automated_data_lineage.png b/images/sqlflow_automated_data_lineage.png new file mode 100644 index 0000000..484e893 Binary files /dev/null and b/images/sqlflow_automated_data_lineage.png differ diff --git a/images/sqlflow_error_message.png b/images/sqlflow_error_message.png new file mode 100644 index 0000000..38dc20e Binary files /dev/null and b/images/sqlflow_error_message.png differ diff --git a/images/sqlflow_introduce1.png b/images/sqlflow_introduce1.png new file mode 100644 index 0000000..46ee0fd Binary files /dev/null and b/images/sqlflow_introduce1.png differ diff --git a/images/sqlflow_introduce2.png b/images/sqlflow_introduce2.png new file mode 100644 index 0000000..4e8058a Binary files /dev/null and b/images/sqlflow_introduce2.png differ diff --git a/images/sqlflow_job_database.png b/images/sqlflow_job_database.png new file mode 100644 index 0000000..edeca0d Binary files /dev/null and b/images/sqlflow_job_database.png differ diff --git a/images/sqlflow_sqlserver_openjson.PNG b/images/sqlflow_sqlserver_openjson.PNG new file mode 100644 index 0000000..9a8c271 Binary files /dev/null and b/images/sqlflow_sqlserver_openjson.PNG differ diff --git a/images/sqlflow_tutorial_101.gif b/images/sqlflow_tutorial_101.gif new file mode 100644 index 0000000..7383347 Binary files /dev/null and b/images/sqlflow_tutorial_101.gif differ diff --git a/images/sqlflow_tutorial_diagram.gif b/images/sqlflow_tutorial_diagram.gif new file mode 100644 index 0000000..277174a Binary files /dev/null and b/images/sqlflow_tutorial_diagram.gif differ diff --git a/images/sqlflow_tutorial_settings.gif b/images/sqlflow_tutorial_settings.gif new file mode 100644 index 0000000..a2d8ce7 Binary files /dev/null and b/images/sqlflow_tutorial_settings.gif differ diff --git a/images/sqlflow_userid_secret_step1.png b/images/sqlflow_userid_secret_step1.png new file mode 100644 index 0000000..32e5e58 Binary files /dev/null and b/images/sqlflow_userid_secret_step1.png differ diff --git a/images/sqlflow_userid_secret_step2.png b/images/sqlflow_userid_secret_step2.png new file mode 100644 index 0000000..0b7db54 Binary files /dev/null and b/images/sqlflow_userid_secret_step2.png differ diff --git a/images/sqlflow_web_ui_control.png b/images/sqlflow_web_ui_control.png new file mode 100644 index 0000000..132995c Binary files /dev/null and b/images/sqlflow_web_ui_control.png differ diff --git a/images/sqlflow_web_ui_join.png b/images/sqlflow_web_ui_join.png new file mode 100644 index 0000000..30838cc Binary files /dev/null and b/images/sqlflow_web_ui_join.png differ diff --git a/install_sqlflow.md b/install_sqlflow.md new file mode 100644 index 0000000..6b476d2 --- /dev/null +++ b/install_sqlflow.md @@ -0,0 +1,420 @@ +- [Instructions on how to install SQLFlow on your own server.](#instructions-on-how-to-install-sqlflow-on-your-own-server) + * [Prerequisites](#prerequisites) + * [Setup Environment (Ubuntu for example)](#setup-environment--ubuntu-for-example-) + * [Upload Files](#upload-files) + * [Nginx Reverse Proxy](#nginx-reverse-proxy) + * [Customize the port](#customize-the-port) + + [1. Default port](#1-default-port) + + [2. Modify the web port](#2-modify-the-web-port) + + [3. Modify java service port](#3-modify-java-service-port) + * [Start Backend Services](#start-backend-services) + * [Start Frontend Services](#start-frontend-services) + * [Gudu SQLFlow License file](#gudu-sqlflow-license-file) + * [Backend Services Configuration](#backend-services-configuration) + * [Sqlflow client api call](#sqlflow-client-api-call) + * [Trouble Shooting](#trouble-shooting) + + [1. Failed to get license info.](#1-failed-to-get-license-info) + + [2. Config nginx on RHEL: Redhat linux](#2-config-nginx-on-rhel-redhat-linux) + + [3. Get license fail: 502 Bad Gateway](#3-get-license-fail-502-bad-gateway) + + + +# Instructions on how to install SQLFlow on your own server. + +SQLFow was comprised of two parts: frontend and backend. +The frontend and backend can be installed on the same server, or they can be installed on two different servers seperately. + +## Prerequisites +- [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- A linux server with at least 8GB memory (ubuntu 20.04 is recommended). +- Java 8 +- Nginx web server. +- Port needs to be opened. (80, 8761,8081,8083. Only 80 port need to be opened if you setup the nginx reverse proxy as mentioned in the below) + +## Setup Environment (Ubuntu for example) + sudo apt-get update + sudo apt-get install nginx -y + sudo apt-get install default-jre -y + +CentOS +- [How To Install Nginx on CentOS](https://www.digitalocean.com/community/tutorials/how-to-install-nginx-on-centos-7) +- [How To Install Java on CentOS ](https://www.digitalocean.com/community/tutorials/how-to-install-java-on-centos-and-fedora) + + +Mac +- [Install SQLFlow on Mac](install_sqlflow_on_mac.md) + +Windows +- [Install SQLFlow on Windows](install_sqlflow_on_windows.md) + +## Upload Files + +create a directory : + +```bash +# it must be created start with root path +sudo mkdir -p /wings/sqlflow +``` + +upload your zip file including backend and frontend file to `sqlflow` folder, and unzip like this : +```bash +unzip sqlflow.zip +``` + +You should get files organized like this: + +``` +/wings/ +└── sqlflow + ├── backend + │ ├── bin + │ │ ├── backend.bat + │ │ ├── backend.sh + │ │ ├── eureka.bat + │ │ ├── eureka.sh + │ │ ├── eureka.vbs + │ │ ├── gspLive.bat + │ │ ├── gspLive.sh + │ │ ├── gspLive.vbs + │ │ ├── monitor.bat + │ │ ├── monitor.sh + │ │ ├── sqlservice.bat + │ │ ├── sqlservice.sh + │ │ ├── sqlservice.vbs + │ │ ├── stop.bat + │ │ ├── stop.sh + │ ├── lib + │ │ ├── eureka.jar + │ │ ├── gspLive.jar + │ │ ├── sqlservice.jar + │ ├── conf + │ │ ├── gudu_sqlflow_license.txt + │ │ ├── gudu_sqlflow.conf + │ ├── data + │ │ ├── job + │ │ │ ├── task + │ │ │ ├── {userid} + │ │ ├── schema + │ │ ├── session + │ │ ├── version + │ ├── log + │ ├── tmp + │ │ └── cache + └── frontend + ├── config.public.json + ├── images + │   ├── check.svg + │   ├── Join.svg + │   ├── pic_Not logged in.png + │   └── visualize.svg + ├── index.********************.css + ├── index.********************.css + ├── index.********************.css + ├── index.********************.css + └── index.html + └── lang + ├── page.*********************.js + ├── page.*********************.js + ├── page.*********************.js + ├── page.*********************.js + ├── public.*********************.js + ├── widget + │   ├── index.js + │   ├── sqlflow-library.version.css + │   └── sqlflow-library.version.js +``` + +set folder permissions : + +```bash +sudo chmod -R 755 /wings/sqlflow +``` + +## Nginx Reverse Proxy + +**1. Config Nginx** + +open your nginx configuration file ( at `/etc/nginx/sites-available/default` under ubuntu ), add a server : + +```nginx +server { + listen 80 default_server; + listen [::]:80 default_server; + + root /wings/sqlflow/frontend/; + index index.html; + + location ~* ^/index.html { + add_header X-Frame-Options deny; # remove this line if you want embed sqlflow in iframe + add_header Cache-Control no-store; + } + + location / { + try_files $uri $uri/ =404; + } + + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } +} +``` +note that `8081` in `proxy_pass http://127.0.0.1:8081/` should be the same as gspLive.jar's port. + +`/api` is mapped to `http://127.0.0.1:8081` in the above configration. This is useful if you +company doesn't allow access `8081` port from the external. + +**2. modify frontend configuration file config.private.json** + +- Open the configration file "/wings/sqlflow/frontend/config.private.json" +- Modify the **ApiPrefix** attribute +``` + "ApiPrefix": "/api" +``` + +## Customize the port + +If you don't want to change the default service port, just ignore this section, +otherwise, please act as the following instructions. + +### 1. Default port +1. Web port is `80` +2. SQLFlow backend service port: + +| File | Port | +| -------------- | ---- | +| eureka.jar | 8761 | +| gspLive.jar | 8081 | +| sqlservice.jar | 8083 | + +### 2. Modify the web port +Change the default web port from `80` to `9000` (or any port you like). + +![sqlflow-install-customize-web-port](/images/sqlflow-install-customize-web-port.png) + +### 3. Modify java service port +Change the default gspLive port from `8081` to `9001`(or any port you like). + +1. Change the port in nginx config file +![sqlflow-install-customize-gsplive-port-nginx](/images/sqlflow-install-customize-gsplive-port-nginx.png) + +2. Change the port in gspLive.sh(gspLive.bat) +![sqlflow-install-customize-port-gsplive](/images/sqlflow-install-customize-port-gsplive.png) + + +## Start Backend Services + +start service in background: + +```bash + sudo /wings/sqlflow/backend/bin/backend.sh +``` + +please allow 3-5 minutes to start the service. + +use `ps -ef|grep java` to check those 3 processing are running. + +``` +ubuntu 11047 1 0 Nov02 ? 00:04:44 java -server -jar eureka.jar +ubuntu 11076 1 0 Nov02 ? 00:04:11 java -server -Xmn512m -Xms2g -Xmx2g -Djavax.accessibility.assistive_technologies= -jar sqlservice.jar +ubuntu 11114 1 0 Nov02 ? 00:05:17 java -server -jar gspLive.jar +``` + + +## Start Frontend Services + +start your nginx : + +```bash +sudo service nginx start +``` + +or reload : + +```bash +sudo nginx -s reload +``` + +open http://yourdomain.com/ to see the SQLFlow. + +open http://yourdomain.com/api/gspLive_backend/doc.html?lang=en to see the Restful API documention. +OR + +open http://yourdomain.com:8081/gspLive_backend/doc.html?lang=en to see the Restful API documention. + + +## Gudu SQLFlow License file +If this is the first time you setup the Gudu SQLFlow on a new machine, +then, you will see this license UI: +![gudu sqlflow license ui](/images/gudu-sqlflow-license.png) + +1. You send us the Gudu SQLFlow Id (6 characters in red). +2. We will generate a license file for you based on this id. +3. You upload the license file by click the "upload license file" link. + +## Backend Services Configuration + +sqlflow provides several optioins to control the service analysis logic. Open the sqlservice configuration file(conf/gudu_sqlflow.conf) + +* **relation_limit**: default value is 2000. When the count of selected object relations is greater than relation_limit, +sqlflow will fallback to the simple mode, ignore all the record sets. +If the relations of simple mode are still greater than relation_limit, sqlflow will only show the summary information. + +* **big_sql_size**: default value is 4096. If the sql length is greater than big_sql_size, sqlflow submit the sql in the work queue and execute it. +If the work queue is full, sqlflow throws an exception and return error message "Sorry, the service is busy. Please try again later." + + +## Sqlflow client api call + +See [sqlflow client api call][1] + +1. Get userId from gudu_sqlflow.conf + - Open the configration file "/wings/sqlflow/backend/conf/gudu_sqlflow.conf" + - The value of anonymous_user_id field is webapi userId + ```bash + anonymous_user_id=xxx + ``` + - **Note:** on-promise mode, webapi call doesn't need the token parameter + +2. Test webapi by curl + + * test sql: + ```sql + select name from user + ``` + + * curl command: + ```bash + curl -X POST "http://yourdomain.com/api/gspLive_backend/sqlflow/generation/sqlflow" -H "accept:application/json;charset=utf-8" -F "userId=YOUR USER ID HERE" -F "dbvendor=dbvoracle" -F "sqltext=select name from user" + ``` + + * response: + ```json + { + "code": 200, + "data": { + "dbvendor": "dbvoracle", + "dbobjs": [ + ... + ], + "relations": [ + ... + ] + }, + "sessionId": ... + } + ``` + * If the code returns **401**, please check the userId is set or the userId is valid. + + +[1]: https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api_full.md#webapi + +## Trouble Shooting +### 1. Failed to get license info. + +![sqlflow-install-failed-to-get-license-info](/images/sqlflow-install-failed-to-get-license-info.png) + +If you see this error, just wait another 3-5 minutes to wait the backend service startup successfully +and refresh the web page. + +Or, this issue may caused by the browser cache, just use `Incognito mode` to access the Sqlflow page and +clear the cache. + + +### 2. Config nginx on RHEL: Redhat linux + +a) Type: vim /etc/nginx/nginx.conf and change the server section of the conf file with below configurations +``` +server { + + listen 80 default_server; + + listen [::]:80 default_server; + + server_name _; + + #root /usr/share/nginx/html; + + root /wings/sqlflow/frontend/; + + index index.html + + # Load configuration files for the default server block. + + include /etc/nginx/default.d/*.conf; + + + + location / { + + try_files $uri $uri/ =404; + + } + + + + error_page 404 /404.html; + + location = /40x.html { + + } + + + + error_page 500 502 503 504 /50x.html; + + location = /50x.html { + + } + + location ~* ^/index.html { + + add_header X-Frame-Options deny; # remove this line if you want to embed SQLFlow in iframe + + add_header Cache-Control no-store; + + } + + location /api/ { + + proxy_pass http://127.0.0.1:8081/; + + proxy_connect_timeout 600s ; + + proxy_read_timeout 600s; + + proxy_send_timeout 600s; + + + + proxy_set_header Host $host; + + proxy_set_header X-Real-IP $remote_addr; + + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_set_header User-Agent $http_user_agent; + + } + +} +``` + +b) Configure selinux to permission by going to: vi /etc/selinux/configure --> SELinux status to = permissive + + +### 3. Get license fail: 502 Bad Gateway + +![gudu sqlflow 502 bad gateway](/images/sqlflow-install-502-bad-gateway.png) + +If you find this error, this is because the port that is needed by the SQLFlow is already used by another application, +please configure the SQLFlow to [use another port](#customize-the-port). + +Or, the Gudu SQLFlow backend service is not started. Please check how to [start the backend and verify the status](#start-backend-services). diff --git a/install_sqlflow_on_mac.md b/install_sqlflow_on_mac.md new file mode 100644 index 0000000..152bdf8 --- /dev/null +++ b/install_sqlflow_on_mac.md @@ -0,0 +1,297 @@ +# Install SQLFlow on your own Mac + +SQLFow was comprised of two parts: frontend and backend. The frontend and backend can be installed on the same server, or they can be installed on two different servers seperately. + +## Prerequisites +- [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- Java 8 +- Nginx web server. +- Port needs to be opened. (80, 8761,8081,8083. Only 80 port need to be opened if you setup the nginx reverse proxy as mentioned in the below) +- At least 8GB memory + +## Setup Environment + +* [Java setup environment link](https://mkyong.com/java/how-to-set-java_home-environment-variable-on-mac-os-x/) + +``` +# setup java environment +echo export "JAVA_HOME=\$(/usr/libexec/java_home)" >> ~/.bash_profile +source ~/.bash_profile +``` + +## Install Nginx + +- [How To Install Nginx on Mac](https://medium.com/@ThomasTan/installing-nginx-in-mac-os-x-maverick-with-homebrew-d8867b7e8a5a) + +## Upload Files + +create a directory : + +``` +# example you can use other path +sudo mkdir -p /wings/sqlflow +``` + +upload your backend and frontend file to `sqlflow` folder, like this : + +``` +/wings/ +└── sqlflow + ├── backend + │ ├── bin + │ │ ├── backend.sh + │ │ ├── stop.sh + │ │ ├── monitor.sh + │ │ ├── sqlservice.sh + │ │ ├── gspLive.sh + │ │ ├── eureka.sh + │ │ ├── backend.bat + │ │ ├── stop.bat + │ │ ├── monitor.bat + │ │ ├── sqlservice.bat + │ │ ├── gspLive.bat + │ │ ├── eureka.bat + │ │ ├── sqlservice.vbs + │ │ ├── gspLive.vbs + │ │ ├── eureka.vbs + │ ├── lib + │ │ ├── sqlservice.jar + │ │ ├── gspLive.jar + │ │ ├── eureka.jar + │ ├── conf + │ │ ├── gudu_sqlflow_license.txt + │ │ ├── gudu_sqlflow.conf + │ ├── data + │ │ ├── job + │ │ │ ├── task + │ │ │ ├── {userid} + │ │ ├── session + │ ├── log + │ │ ├── sqlservice.log + │ │ ├── gspLive.log + │ │ ├── eureka.log + │ │ ├── slow (slow query records) + │ │ ├── sqlflow (sqlflow access records) + │ ├── tmp + │ │ └── cache + └── frontend + ├── 1.app.b95fd285b4e8a1af563a.js + ├── 1.index.b95fd285b4e8a1af563a.css + ├── app.b95fd285b4e8a1af563a.js + ├── config.private.json + ├── font + │ ├── Roboto-Regular.ttf + │ ├── segoeui-light.woff2 + │ └── segoeui-semilight.woff2 + ├── images + │ ├── check.svg + │ ├── Join.svg + │ ├── pic_Not logged in.png + │ └── visualize.svg + ├── index.b95fd285b4e8a1af563a.css + └── index.html +``` + +## set scripts permissions : + +``` +chmod +x /wings/sqlflow/backend/bin +``` + +## Start Backend Services + +If your computer has more than 8G of memory, you can change the boot parameters to recommended + +Please use the gspLive.sh, eureka.sh and sqlservice.sh under [mac](mac) directory instead of the original one. + +* /wings/sqlflow/backend/bin/gspLive.sh + +``` +# defult and less than or equal to 8G recommended +heapsize="2g"; +# greater than 8G recommended +# heapsize="3g"; + +``` + +* /wings/sqlflow/backend/bin/eureka.sh + +``` +# defult and less than or equal to 8G recommended +heapsize="256m"; +# greater than 8G recommended +# heapsize="512m"; +``` + +* /wings/sqlflow/backend/bin/sqlservice.sh + +``` +# defult and less than or equal to 8G recommended +heapsize="4g"; +# greater than 8G recommended +# heapsize="10g"; +``` + +start service in background: + +``` +sh /wings/sqlflow/backend/bin/backend.sh +``` + +please allow 1-2 minutes to start the service. + +use `jps` to check those 3 processing are running. + +``` +58497 sqlservice.jar +58516 gspLive.jar +58477 eureka.jar +``` + +**Java service port** + +| File | Port | +| -------------- | ---- | +| eureka.jar | 8761 | +| gspLive.jar | 8081 | +| sqlservice.jar | 8083 | + +## Nginx Reverse Proxy + +If we set the reverse proxy path of gspLive restful service to /api + +**1. Config Nginx** + +open your nginx configuration file ( at `/usr/local/etc/ngin/nginx.conf` ), add a server : + +``` +server { + listen 80 default_server; + listen [::]:80 default_server; + + + root /wings/sqlflow/frontend/; + index index.html; + + location ~* ^/index.html { + add_header X-Frame-Options deny; + add_header Cache-Control no-store; + } + + location / { + try_files $uri $uri/ =404; + } + + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } +} +``` + +**2. modify frontend configuration file config.private.json** + +- Open the configration file "/wings/sqlflow/frontend/config.private.json" +- Modify the **ApiPrefix** attribute + +``` + "ApiPrefix": "/api" +``` + +### Start Frontend Services + +start your nginx : + +``` +sudo nginx +``` + +or reload : + +``` +sudo nginx -s reload +``` + +open http://yourdomain.com/ to see the SQLFlow. + +open http://yourdomain.com/api/gspLive_backend/doc.html?lang=en to see the Restful API documention. + +## Sqlflow client api call + +See [sqlflow client api call](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api_full.md#webapi) + +1. Get userId from gudu_sqlflow.conf + +- Open the configration file "/wings/sqlflow/backend/conf/gudu_sqlflow.conf" +- The value of anonymous_user_id field is webapi userId + +``` + anonymous_user_id=xxx +``` + +- **Note:** on-promise mode, webapi call doesn't need the token parameter + +1. Test webapi by curl + + - test sql: + + ``` + select name from user + ``` + + - curl command: + + ``` + curl -X POST "http://yourdomain.com/api/gspLive_backend/sqlflow/generation/sqlflow" -H "accept:application/json;charset=utf-8" -F "userId=YOUR USER ID HERE" -F "dbvendor=dbvoracle" -F "sqltext=select name from user" + ``` + + - response: + + ``` + { + "code": 200, + "data": { + "dbvendor": "dbvoracle", + "dbobjs": [ + ... + ], + "relations": [ + ... + ] + }, + "sessionId": ... + } + ``` + + - If the code returns **401**, please check the userId is set or the userId is valid. + +## How to use grabit submit to local SQLFlow + +1. Get userId from gudu_sqlflow.conf + +* Open the configration file "/wings/sqlflow/backend/conf/gudu_sqlflow.conf" + +* The value of anonymous_user_id field is webapi userId + + ``` + anonymous_user_id=xxx + ``` + +2. Please Input SqlFlow Connect Information tab + + ``` + Server: http://yourdomain.com/api eg: http://localhost/api + ServerPort: doesn't need + UserId: please see No.1 + User Secret: Just fill in any one + ``` + + + diff --git a/install_sqlflow_on_windows.md b/install_sqlflow_on_windows.md new file mode 100644 index 0000000..02997d0 --- /dev/null +++ b/install_sqlflow_on_windows.md @@ -0,0 +1,139 @@ +### Prerequisites +- [SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) +- Server with at least 8GB memory +- install JDK1.8 or higher + SET JAVA_HOME variable, and then add %JAVA_HOME%\bin to the path variable + +- install Nginx for windows + Download the Nginx Windows version here: http://nginx.org/en/docs/windows.html + +### unzip SQLFlow file +- create a dirctory: c:\wings\sqlflow +- unzip SQLFlow install package to c:\wings\sqlflow, you will get 2 directories like: + 1. c:\wings\sqlflow\backend + 2. c:\wings\sqlflow\frontend + + +### start SQLFlow backend +- Open a dos command windows +- cd c:\wings\sqlflow\backend\bin +- run monitor.bat +- please wait 3-5 minutes to allow the SQLFlow service to start completely. + +### Nginx Reverse Proxy + +If we set the reverse proxy path of gspLive restful service to /api + +**1. config Nginx** +- enter conf directory where Nginx is installed such as Nginx-1.19.4\conf +- modify the Nginx.conf, replace the server section in nginx.conf with the following code: +``` + server { + + listen 80 default_server; + + listen [::]:80 default_server; + + root C:\wings\sqlflow\frontend; + + index index.html; + + location ~* ^/index.html { + add_header X-Frame-Options deny; + add_header Cache-Control no-store; + } + + location / { + try_files $uri $uri/ =404; + } + + location /api/ { + proxy_pass http://127.0.0.1:8081/; + proxy_connect_timeout 600s ; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header User-Agent $http_user_agent; + } + } + +``` + +Please make sure `C:\wings\sqlflow\frontend` is where the SQLFlow frontend is installed, +otherwise, please change `C:\wings\sqlflow\frontend` to the path where the SQLFlow frontend is located. + +**2. modify frontend configuration file config.private.json** + +- Open the configration file "C:\wings\sqlflow\frontend\config.private.json" +- Modify the **ApiPrefix** attribute +``` + "ApiPrefix": "/api" +``` + +### start Nginx +- Open a dos command window +- cd the directory where Nginx is installed +- run just nginx.exe + +### SQLFlow is ready +Just Open the browser and enter the localhost or IP where the SQLFlow is installed. + +open http://yourIp/ to see the SQLFlow. + +open http://yourIp/api/gspLive_backend/doc.html?lang=en to see the Restful API documention. + +### stop the SQLFlow +- close the window where the monitor.sh is running. +- cd c:\wings\sqlflow\backend\bin +- run stop.bat + +### Sqlflow client api call + +See [sqlflow client api call][1] + +1. Get userId from gudu_sqlflow.conf + - Open the configration file "c:\wings\sqlflow\backend\conf\gudu_sqlflow.conf" + - The value of anonymous_user_id field is webapi userId + ```bash + anonymous_user_id=xxx + ``` + - **Note:** on-promise mode, webapi call doesn't need the token parameter + +2. Test webapi by curl + + * test sql: + ```sql + select name from user + ``` + + * curl command: + ```bash + curl -X POST "http://yourIp/api/gspLive_backend/sqlflow/generation/sqlflow" -H "accept:application/json;charset=utf-8" -F "userId=YOUR USER ID HERE" -F "dbvendor=dbvoracle" -F "sqltext=select name from user" + ``` + + * response: + ```json + { + "code": 200, + "data": { + "dbvendor": "dbvoracle", + "dbobjs": [ + ... + ], + "relations": [ + ... + ] + }, + "sessionId": ... + } + ``` + * If the code returns **401**, please check the userId is set or the userId is valid. + +### troubleshooting +- make sure the window hostname doesn't include the underscore symbol (_), otherwise, the service will not work properly. + please change it to minus symbol (-) + +[1]: https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api_full.md#webapi diff --git a/integration/atlas/models/2000-RDBMS/2010-rdbms_model.json b/integration/atlas/models/2000-RDBMS/2010-rdbms_model.json new file mode 100644 index 0000000..3d784bf --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/2010-rdbms_model.json @@ -0,0 +1,458 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "rdbms_instance", + "description": "Instance that the rdbms server is running on", + "superTypes": ["DataSet"], + "serviceType": "rdbms", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "rdbms_type", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "platform", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "cloudOrOnPrem", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "hostname", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "port", + "typeName": "int", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "protocol", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "contact_info", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "rdbms_db", + "description": "a database (schema) in an rdbms", + "superTypes": ["DataSet"], + "serviceType": "rdbms", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "prodOrOther", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "contact_info", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "rdbms_table", + "description": "a table in an rdbms database (schema)", + "superTypes": ["DataSet"], + "serviceType": "rdbms", + "typeVersion": "1.2", + "options": { + "schemaElementsAttribute": "columns" + }, + "attributeDefs": [ + { + "name": "name_path", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "createTime", + "typeName": "date", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "contact_info", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "rdbms_column", + "description": "a column in an rdbms table", + "superTypes": ["DataSet"], + "serviceType": "rdbms", + "typeVersion": "1.2", + "options": { + "schemaAttributes": "[\"name\", \"description\", \"owner\", \"data_type\", \"comment\", \" isPrimaryKey\", \" isNullable\"]" + }, + "attributeDefs": [ + { + "name": "data_type", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "length", + "typeName": "int", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "default_value", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "isNullable", + "typeName": "boolean", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "isPrimaryKey", + "typeName": "boolean", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "rdbms_index", + "description": "An index on an RDBMS table", + "superTypes": ["DataSet"], + "serviceType": "rdbms", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "index_type", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "isUnique", + "typeName": "boolean", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "rdbms_foreign_key", + "description": null, + "superTypes": ["DataSet"], + "serviceType": "rdbms", + "typeVersion": "1.1", + "attributeDefs": [ + ] + } + ], + "relationshipDefs": [ + { + "name": "rdbms_instance_databases", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__rdbms_instance.databases", + "endDef1": { + "type": "rdbms_instance", + "name": "databases", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_db", + "name": "instance", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_db_tables", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__rdbms_db.tables", + "endDef1": { + "type": "rdbms_db", + "name": "tables", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_table", + "name": "db", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_table_columns", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__rdbms_table.columns", + "endDef1": { + "type": "rdbms_table", + "name": "columns", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_column", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_table_indexes", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__rdbms_table.indexes", + "endDef1": { + "type": "rdbms_table", + "name": "indexes", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_index", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_index_columns", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__rdbms_index.columns", + "endDef1": { + "type": "rdbms_index", + "name": "columns", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_column", + "name": "indexes", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_table_foreign_key", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__rdbms_table.foreign_keys", + "endDef1": { + "type": "rdbms_table", + "name": "foreign_keys", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_foreign_key", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_foreign_key_key_columns", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__rdbms_foreign_key.key_columns", + "endDef1": { + "type": "rdbms_foreign_key", + "name": "key_columns", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_column", + "name": "key_column_references", + "isContainer": false, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_foreign_key_table_references", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__rdbms_foreign_key.references_table", + "endDef1": { + "type": "rdbms_foreign_key", + "name": "references_table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_table", + "name": "foreign_key_references", + "isContainer": false, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "rdbms_foreign_key_column_references", + "serviceType": "rdbms", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__rdbms_foreign_key.references_columns", + "endDef1": { + "type": "rdbms_foreign_key", + "name": "references_columns", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "rdbms_column", + "name": "foreign_key_references", + "isContainer": false, + "cardinality": "SET" + }, + "propagateTags": "NONE" + } + ] +} diff --git a/integration/atlas/models/2000-RDBMS/2030-snowflake_model.json b/integration/atlas/models/2000-RDBMS/2030-snowflake_model.json new file mode 100644 index 0000000..d44acea --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/2030-snowflake_model.json @@ -0,0 +1,573 @@ +{ + "enumDefs": [ + { + "name": "snowflake_principal_type", + "serviceType": "snowflake", + "typeVersion": "1.0", + "elementDefs": [ + { + "ordinal": 1, + "value": "USER" + }, + { + "ordinal": 2, + "value": "ROLE" + }, + { + "ordinal": 3, + "value": "GROUP" + } + ] + } + ], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "snowflake_process", + "superTypes": [ + "Process" + ], + "serviceType": "snowflake", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "userName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "operationType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryPlan", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryId", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "recentQueries", + "typeName": "array", + "cardinality": "LIST", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "includeInNotification": true, + "isUnique": false + }, + { + "name": "queryGraph", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "snowflake_table", + "superTypes": [ + "DataSet" + ], + "serviceType": "snowflake", + "typeVersion": "1.1", + "options": { + "schemaElementsAttribute": "columns" + }, + "attributeDefs": [ + { + "name": "createTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "lastAccessTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "comment", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "retention", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "aliases", + "typeName": "array", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "viewOriginalText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "viewExpandedText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "tableType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "temporary", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "snowflake_db", + "superTypes": [ + "Asset" + ], + "serviceType": "snowflake", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "includeInNotification": true, + "isUnique": false + }, + { + "name": "location", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "ownerType", + "typeName": "snowflake_principal_type", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "snowflake_column", + "superTypes": [ + "DataSet" + ], + "serviceType": "snowflake", + "typeVersion": "1.3", + "options": { + "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]" + }, + "attributeDefs": [ + { + "name": "type", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "comment", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "position", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name" : "snowflake_column_lineage", + "superTypes" : [ + "Process" + ], + "serviceType": "snowflake", + "typeVersion" : "1.0", + "attributeDefs" : [ + { + "name": "depenendencyType", + "typeName": "string", + "cardinality" : "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "expression", + "typeName": "string", + "cardinality" : "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name" : "snowflake_process_execution", + "superTypes" : [ + "ProcessExecution" + ], + "serviceType": "snowflake", + "typeVersion" : "1.1", + "attributeDefs" : [ + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "userName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryGraph", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "queryId", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryPlan", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "hostName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + } + ], + "options": { + "displayTextAttribute": "queryText" + } + }, + { + "name": "snowflake_db_ddl", + "superTypes": [ + "ddl" + ], + "serviceType": "snowflake", + "typeVersion": "1.0", + "attributeDefs": [] + }, + { + "name": "snowflake_table_ddl", + "superTypes": [ + "ddl" + ], + "serviceType": "snowflake", + "typeVersion": "1.0", + "attributeDefs": [] + } + ], + "relationshipDefs": [ + { + "name": "snowflake_table_db", + "serviceType": "snowflake", + "typeVersion": "1.2", + "relationshipCategory": "AGGREGATION", + "relationshipLabel": "__snowflake_table.db", + "endDef1": { + "type": "snowflake_table", + "name": "db", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "snowflake_db", + "name": "tables", + "isContainer": true, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_table_columns", + "serviceType": "snowflake", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__snowflake_table.columns", + "endDef1": { + "type": "snowflake_table", + "name": "columns", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "snowflake_column", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_table_partitionkeys", + "serviceType": "snowflake", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__snowflake_table.partitionKeys", + "endDef1": { + "type": "snowflake_table", + "name": "partitionKeys", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "snowflake_column", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_process_column_lineage", + "serviceType": "snowflake", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__snowflake_column_lineage.query", + "endDef1": { + "type": "snowflake_column_lineage", + "name": "query", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "snowflake_process", + "name": "columnLineages", + "isContainer": true, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_process_process_executions", + "serviceType": "snowflake", + "typeVersion": "1.0", + "relationshipCategory": "COMPOSITION", + "endDef1": { + "type": "snowflake_process", + "name": "processExecutions", + "cardinality": "SET", + "isContainer": true + }, + "endDef2": { + "type": "snowflake_process_execution", + "name": "process", + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_table_ddl_queries", + "serviceType": "snowflake", + "typeVersion": "1.0", + "relationshipCategory": "COMPOSITION", + "endDef1": { + "type": "snowflake_table", + "name": "ddlQueries", + "isContainer": true, + "cardinality": "SET" + }, + "endDef2": { + "type": "snowflake_table_ddl", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_db_ddl_queries", + "serviceType": "snowflake", + "typeVersion": "1.0", + "relationshipCategory": "COMPOSITION", + "endDef1": { + "type": "snowflake_db", + "name": "ddlQueries", + "isContainer": true, + "cardinality": "SET" + }, + "endDef2": { + "type": "snowflake_db_ddl", + "name": "db", + "isContainer": false, + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + }, + { + "name": "snowflake_db_location", + "serviceType": "snowflake", + "typeVersion": "1.0", + "relationshipCategory": "ASSOCIATION", + "endDef1": { + "type": "snowflake_db", + "name": "locationPath", + "cardinality": "SINGLE" + }, + "endDef2": { + "type": "Path", + "name": "snowflakeDb", + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + } + ] +} diff --git a/integration/atlas/models/2000-RDBMS/2040-sqlserver_model.json b/integration/atlas/models/2000-RDBMS/2040-sqlserver_model.json new file mode 100644 index 0000000..8624c82 --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/2040-sqlserver_model.json @@ -0,0 +1,573 @@ +{ + "enumDefs": [ + { + "name": "sqlserver_principal_type", + "serviceType": "sqlserver", + "typeVersion": "1.0", + "elementDefs": [ + { + "ordinal": 1, + "value": "USER" + }, + { + "ordinal": 2, + "value": "ROLE" + }, + { + "ordinal": 3, + "value": "GROUP" + } + ] + } + ], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "sqlserver_process", + "superTypes": [ + "Process" + ], + "serviceType": "sqlserver", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "userName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "operationType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryPlan", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryId", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "recentQueries", + "typeName": "array", + "cardinality": "LIST", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "includeInNotification": true, + "isUnique": false + }, + { + "name": "queryGraph", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "sqlserver_table", + "superTypes": [ + "DataSet" + ], + "serviceType": "sqlserver", + "typeVersion": "1.1", + "options": { + "schemaElementsAttribute": "columns" + }, + "attributeDefs": [ + { + "name": "createTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "lastAccessTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "comment", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "retention", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "aliases", + "typeName": "array", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "viewOriginalText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "viewExpandedText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "tableType", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "temporary", + "typeName": "boolean", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "sqlserver_db", + "superTypes": [ + "Asset" + ], + "serviceType": "sqlserver", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "clusterName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "includeInNotification": true, + "isUnique": false + }, + { + "name": "location", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "parameters", + "typeName": "map", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "ownerType", + "typeName": "sqlserver_principal_type", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name": "sqlserver_column", + "superTypes": [ + "DataSet" + ], + "serviceType": "sqlserver", + "typeVersion": "1.3", + "options": { + "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]" + }, + "attributeDefs": [ + { + "name": "type", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "comment", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "position", + "typeName": "int", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name" : "sqlserver_column_lineage", + "superTypes" : [ + "Process" + ], + "serviceType": "sqlserver", + "typeVersion" : "1.0", + "attributeDefs" : [ + { + "name": "depenendencyType", + "typeName": "string", + "cardinality" : "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "expression", + "typeName": "string", + "cardinality" : "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + } + ] + }, + { + "name" : "sqlserver_process_execution", + "superTypes" : [ + "ProcessExecution" + ], + "serviceType": "sqlserver", + "typeVersion" : "1.1", + "attributeDefs" : [ + { + "name": "startTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "endTime", + "typeName": "date", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "userName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryText", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryGraph", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false + }, + { + "name": "queryId", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "queryPlan", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": false, + "isUnique": false + }, + { + "name": "hostName", + "typeName": "string", + "cardinality": "SINGLE", + "isIndexable": true, + "isOptional": false, + "isUnique": false + } + ], + "options": { + "displayTextAttribute": "queryText" + } + }, + { + "name": "sqlserver_db_ddl", + "superTypes": [ + "ddl" + ], + "serviceType": "sqlserver", + "typeVersion": "1.0", + "attributeDefs": [] + }, + { + "name": "sqlserver_table_ddl", + "superTypes": [ + "ddl" + ], + "serviceType": "sqlserver", + "typeVersion": "1.0", + "attributeDefs": [] + } + ], + "relationshipDefs": [ + { + "name": "sqlserver_table_db", + "serviceType": "sqlserver", + "typeVersion": "1.2", + "relationshipCategory": "AGGREGATION", + "relationshipLabel": "__sqlserver_table.db", + "endDef1": { + "type": "sqlserver_table", + "name": "db", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "sqlserver_db", + "name": "tables", + "isContainer": true, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_table_columns", + "serviceType": "sqlserver", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__sqlserver_table.columns", + "endDef1": { + "type": "sqlserver_table", + "name": "columns", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "sqlserver_column", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_table_partitionkeys", + "serviceType": "sqlserver", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__sqlserver_table.partitionKeys", + "endDef1": { + "type": "sqlserver_table", + "name": "partitionKeys", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "sqlserver_column", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_process_column_lineage", + "serviceType": "sqlserver", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__sqlserver_column_lineage.query", + "endDef1": { + "type": "sqlserver_column_lineage", + "name": "query", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "sqlserver_process", + "name": "columnLineages", + "isContainer": true, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_process_process_executions", + "serviceType": "sqlserver", + "typeVersion": "1.0", + "relationshipCategory": "COMPOSITION", + "endDef1": { + "type": "sqlserver_process", + "name": "processExecutions", + "cardinality": "SET", + "isContainer": true + }, + "endDef2": { + "type": "sqlserver_process_execution", + "name": "process", + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_table_ddl_queries", + "serviceType": "sqlserver", + "typeVersion": "1.0", + "relationshipCategory": "COMPOSITION", + "endDef1": { + "type": "sqlserver_table", + "name": "ddlQueries", + "isContainer": true, + "cardinality": "SET" + }, + "endDef2": { + "type": "sqlserver_table_ddl", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_db_ddl_queries", + "serviceType": "sqlserver", + "typeVersion": "1.0", + "relationshipCategory": "COMPOSITION", + "endDef1": { + "type": "sqlserver_db", + "name": "ddlQueries", + "isContainer": true, + "cardinality": "SET" + }, + "endDef2": { + "type": "sqlserver_db_ddl", + "name": "db", + "isContainer": false, + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + }, + { + "name": "sqlserver_db_location", + "serviceType": "sqlserver", + "typeVersion": "1.0", + "relationshipCategory": "ASSOCIATION", + "endDef1": { + "type": "sqlserver_db", + "name": "locationPath", + "cardinality": "SINGLE" + }, + "endDef2": { + "type": "Path", + "name": "sqlserverDb", + "cardinality": "SINGLE" + }, + "propagateTags": "NONE" + } + ] +} diff --git a/integration/atlas/models/2000-RDBMS/2050-mysql_model.json b/integration/atlas/models/2000-RDBMS/2050-mysql_model.json new file mode 100644 index 0000000..4a6fd00 --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/2050-mysql_model.json @@ -0,0 +1,458 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "name": "mysql_instance", + "description": "Instance that the mysql server is running on", + "superTypes": ["DataSet"], + "serviceType": "mysql", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "mysql_type", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "platform", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "cloudOrOnPrem", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "hostname", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "port", + "typeName": "int", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "protocol", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "contact_info", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "mysql_db", + "description": "a database (schema) in an mysql", + "superTypes": ["DataSet"], + "serviceType": "mysql", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "prodOrOther", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "contact_info", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "mysql_table", + "description": "a table in an mysql database (schema)", + "superTypes": ["DataSet"], + "serviceType": "mysql", + "typeVersion": "1.2", + "options": { + "schemaElementsAttribute": "columns" + }, + "attributeDefs": [ + { + "name": "name_path", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "createTime", + "typeName": "date", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "type", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "contact_info", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": 1, + "valuesMaxCount": 1, + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "mysql_column", + "description": "a column in an mysql table", + "superTypes": ["DataSet"], + "serviceType": "mysql", + "typeVersion": "1.2", + "options": { + "schemaAttributes": "[\"name\", \"description\", \"owner\", \"data_type\", \"comment\", \" isPrimaryKey\", \" isNullable\"]" + }, + "attributeDefs": [ + { + "name": "data_type", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": true + }, + { + "name": "length", + "typeName": "int", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "default_value", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "isNullable", + "typeName": "boolean", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "isPrimaryKey", + "typeName": "boolean", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "mysql_index", + "description": "An index on an mysql table", + "superTypes": ["DataSet"], + "serviceType": "mysql", + "typeVersion": "1.1", + "attributeDefs": [ + { + "name": "index_type", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "isUnique", + "typeName": "boolean", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + }, + { + "name": "comment", + "typeName": "string", + "isOptional": true, + "cardinality": "SINGLE", + "isUnique": false, + "isIndexable": false + } + ] + }, + { + "name": "mysql_foreign_key", + "description": null, + "superTypes": ["DataSet"], + "serviceType": "mysql", + "typeVersion": "1.1", + "attributeDefs": [ + ] + } + ], + "relationshipDefs": [ + { + "name": "mysql_instance_databases", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__mysql_instance.databases", + "endDef1": { + "type": "mysql_instance", + "name": "databases", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_db", + "name": "instance", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_db_tables", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__mysql_db.tables", + "endDef1": { + "type": "mysql_db", + "name": "tables", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_table", + "name": "db", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_table_columns", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__mysql_table.columns", + "endDef1": { + "type": "mysql_table", + "name": "columns", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_column", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_table_indexes", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__mysql_table.indexes", + "endDef1": { + "type": "mysql_table", + "name": "indexes", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_index", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_index_columns", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__mysql_index.columns", + "endDef1": { + "type": "mysql_index", + "name": "columns", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_column", + "name": "indexes", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_table_foreign_key", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "COMPOSITION", + "relationshipLabel": "__mysql_table.foreign_keys", + "endDef1": { + "type": "mysql_table", + "name": "foreign_keys", + "isContainer": true, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_foreign_key", + "name": "table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_foreign_key_key_columns", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__mysql_foreign_key.key_columns", + "endDef1": { + "type": "mysql_foreign_key", + "name": "key_columns", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_column", + "name": "key_column_references", + "isContainer": false, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_foreign_key_table_references", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__mysql_foreign_key.references_table", + "endDef1": { + "type": "mysql_foreign_key", + "name": "references_table", + "isContainer": false, + "cardinality": "SINGLE", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_table", + "name": "foreign_key_references", + "isContainer": false, + "cardinality": "SET" + }, + "propagateTags": "NONE" + }, + { + "name": "mysql_foreign_key_column_references", + "serviceType": "mysql", + "typeVersion": "1.2", + "relationshipCategory": "ASSOCIATION", + "relationshipLabel": "__mysql_foreign_key.references_columns", + "endDef1": { + "type": "mysql_foreign_key", + "name": "references_columns", + "isContainer": false, + "cardinality": "SET", + "isLegacyAttribute": true + }, + "endDef2": { + "type": "mysql_column", + "name": "foreign_key_references", + "isContainer": false, + "cardinality": "SET" + }, + "propagateTags": "NONE" + } + ] +} diff --git a/integration/atlas/models/2000-RDBMS/patches/001-rdbms_column_table_add_options.json b/integration/atlas/models/2000-RDBMS/patches/001-rdbms_column_table_add_options.json new file mode 100644 index 0000000..f37d6da --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/patches/001-rdbms_column_table_add_options.json @@ -0,0 +1,26 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_2000_101", + "description": "Add 'schemaAttributes' typeDefOptions to rdbms_column", + "action": "UPDATE_TYPEDEF_OPTIONS", + "typeName": "rdbms_column", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "typeDefOptions": { + "schemaAttributes": "[\"name\", \"description\", \"owner\", \"data_type\", \"comment\", \" isPrimaryKey\", \" isNullable\"]" + } + }, + { + "id": "TYPEDEF_PATCH_2000_102", + "description": "Add 'schemaElementsAttribute' typeDefOptions to rdbms_table", + "action": "UPDATE_TYPEDEF_OPTIONS", + "typeName": "rdbms_table", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "typeDefOptions": { + "schemaElementsAttribute": "columns" + } + } + ] +} \ No newline at end of file diff --git a/integration/atlas/models/2000-RDBMS/patches/002-rdbms_model_add_service_type.json b/integration/atlas/models/2000-RDBMS/patches/002-rdbms_model_add_service_type.json new file mode 100644 index 0000000..2ef1f28 --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/patches/002-rdbms_model_add_service_type.json @@ -0,0 +1,139 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_2000_103", + "description": "Set serviceType 'rdbms' to rdbms_instance", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_instance", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_104", + "description": "Set serviceType 'rdbms' to rdbms_db", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_db", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_105", + "description": "Set serviceType 'rdbms' to rdbms_table", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_table", + "applyToVersion": "1.2", + "updateToVersion": "1.3", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_106", + "description": "Set serviceType 'rdbms' to rdbms_column", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_column", + "applyToVersion": "1.2", + "updateToVersion": "1.3", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_107", + "description": "Set serviceType 'rdbms' to rdbms_index", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_index", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_108", + "description": "Set serviceType 'rdbms' to rdbms_foreign_key", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_foreign_key", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_109", + "description": "Set serviceType 'rdbms' to rdbms_instance_databases", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_instance_databases", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_110", + "description": "Set serviceType 'rdbms' to rdbms_db_tables", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_db_tables", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_111", + "description": "Set serviceType 'rdbms' to rdbms_table_columns", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_table_columns", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_112", + "description": "Set serviceType 'rdbms' to rdbms_table_indexes", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_table_indexes", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_113", + "description": "Set serviceType 'rdbms' to rdbms_index_columns", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_index_columns", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_114", + "description": "Set serviceType 'rdbms' to rdbms_table_foreign_key", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_table_foreign_key", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_115", + "description": "Set serviceType 'rdbms' to rdbms_foreign_key_key_columns", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_foreign_key_key_columns", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_116", + "description": "Set serviceType 'rdbms' to rdbms_foreign_key_table_references", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_foreign_key_table_references", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + }, + { + "id": "TYPEDEF_PATCH_2000_117", + "description": "Set serviceType 'rdbms' to rdbms_foreign_key_column_references", + "action": "SET_SERVICE_TYPE", + "typeName": "rdbms_foreign_key_column_references", + "applyToVersion": "1.0", + "updateToVersion": "1.1", + "serviceType": "rdbms" + } + ] +} \ No newline at end of file diff --git a/integration/atlas/models/2000-RDBMS/patches/003-remove-rdbms-legacy-attributes.json b/integration/atlas/models/2000-RDBMS/patches/003-remove-rdbms-legacy-attributes.json new file mode 100644 index 0000000..990d4e9 --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/patches/003-remove-rdbms-legacy-attributes.json @@ -0,0 +1,108 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_2000_118", + "description": "Remove legacy reference attribute 'databases' from rdbms_instance", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_instance_databases", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_instance.databases", + "relationshipCategory": "COMPOSITION" + } + }, + { + "id": "TYPEDEF_PATCH_2000_119", + "description": "Remove legacy reference attribute 'tables' from rdbms_db", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_db_tables", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_db.tables", + "relationshipCategory": "COMPOSITION" + } + }, + { + "id": "TYPEDEF_PATCH_2000_120", + "description": "Remove legacy reference attribute 'columns' from rdbms_table", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_table_columns", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_table.columns", + "relationshipCategory": "COMPOSITION" + } + }, + { + "id": "TYPEDEF_PATCH_2000_121", + "description": "Remove legacy reference attribute 'indexes' from rdbms_table", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_table_indexes", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_table.indexes", + "relationshipCategory": "COMPOSITION" + } + }, + { + "id": "TYPEDEF_PATCH_2000_122", + "description": "Remove legacy reference attribute 'foreign_keys' from rdbms_table", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_table_foreign_key", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_table.foreign_keys", + "relationshipCategory": "COMPOSITION" + } + }, + { + "id": "TYPEDEF_PATCH_2000_123", + "description": "Remove legacy reference attribute 'columns' from rdbms_index", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_index_columns", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_index.columns" + } + }, + { + "id": "TYPEDEF_PATCH_2000_124", + "description": "Remove legacy reference attribute 'key_columns' from rdbms_foreign_key", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_foreign_key_key_columns", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_foreign_key.key_columns" + } + }, + { + "id": "TYPEDEF_PATCH_2000_125", + "description": "Remove legacy reference attribute 'references_table' from rdbms_foreign_key", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_foreign_key_table_references", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_foreign_key.references_table" + } + }, + { + "id": "TYPEDEF_PATCH_2000_126", + "description": "Remove legacy reference attribute 'references_columns' from rdbms_foreign_key", + "action": "REMOVE_LEGACY_REF_ATTRIBUTES", + "typeName": "rdbms_foreign_key_column_references", + "applyToVersion": "1.1", + "updateToVersion": "1.2", + "params": { + "relationshipLabel": "__rdbms_foreign_key.references_columns" + } + } + ] +} diff --git a/integration/atlas/models/2000-RDBMS/patches/004-add_searchweight.json b/integration/atlas/models/2000-RDBMS/patches/004-add_searchweight.json new file mode 100644 index 0000000..bf060d8 --- /dev/null +++ b/integration/atlas/models/2000-RDBMS/patches/004-add_searchweight.json @@ -0,0 +1,126 @@ +{ + "patches": [ + { + "id": "TYPEDEF_PATCH_2000_127", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_instance", + "applyToVersion": "1.2", + "updateToVersion": "1.3", + "attributeName": "rdbms_type", + "params": { + "searchWeight": 8 + } + }, { + "id": "TYPEDEF_PATCH_2000_128", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_instance", + "applyToVersion": "1.3", + "updateToVersion": "1.4", + "attributeName": "platform", + "params": { + "searchWeight": 8 + } + }, { + "id": "TYPEDEF_PATCH_2000_129", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_instance", + "applyToVersion": "1.4", + "updateToVersion": "1.5", + "attributeName": "contact_info", + "params": { + "searchWeight": 8 + } + }, { + "id": "TYPEDEF_PATCH_2000_130", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_instance", + "applyToVersion": "1.5", + "updateToVersion": "1.6", + "attributeName": "hostname", + "params": { + "searchWeight": 9 + } + }, { + "id": "TYPEDEF_PATCH_2000_131", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_instance", + "applyToVersion": "1.6", + "updateToVersion": "1.7", + "attributeName": "comment", + "params": { + "searchWeight": 9 + } + }, { + "id": "TYPEDEF_PATCH_2000_132", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_db", + "applyToVersion": "1.2", + "updateToVersion": "1.3", + "attributeName": "contact_info", + "params": { + "searchWeight": 8 + } + }, { + "id": "TYPEDEF_PATCH_2000_133", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_table", + "applyToVersion": "1.3", + "updateToVersion": "1.4", + "attributeName": "name_path", + "params": { + "searchWeight": 5 + } + }, { + "id": "TYPEDEF_PATCH_2000_134", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_table", + "applyToVersion": "1.4", + "updateToVersion": "1.5", + "attributeName": "comment", + "params": { + "searchWeight": 9 + } + }, { + "id": "TYPEDEF_PATCH_2000_135", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_table", + "applyToVersion": "1.5", + "updateToVersion": "1.6", + "attributeName": "contact_info", + "params": { + "searchWeight": 8 + } + }, { + "id": "TYPEDEF_PATCH_2000_136", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_column", + "applyToVersion": "1.3", + "updateToVersion": "1.4", + "attributeName": "comment", + "params": { + "searchWeight": 9 + } + }, { + "id": "TYPEDEF_PATCH_2000_137", + "description": "Adding search Weights", + "action": "UPDATE_ATTRIBUTE_METADATA", + "typeName": "rdbms_index", + "applyToVersion": "1.2", + "updateToVersion": "1.3", + "attributeName": "comment", + "params": { + "searchWeight": 9 + } + } + ] +} diff --git a/json_output_objects_coordinate.md b/json_output_objects_coordinate.md new file mode 100644 index 0000000..74607c8 --- /dev/null +++ b/json_output_objects_coordinate.md @@ -0,0 +1,136 @@ +## Get SQL Information By SQLFLow Coordinate + +### SQLInfo +When the sqlflow analyzing sql has been finished, it recorded some sql information, we can use it to locate database object position. + +```java +public class SqlInfo { + private String fileName; + private String filePath; + private String sql; + private int originIndex; + private int index; + private String group; + private int originLineStart; + private int originLineEnd; + private int lineStart; + private int lineEnd; + private String hash; +} +``` + +Each sql file matches a SqlInfo object, and the map key is "hash" property. + +Sqlflow provides a tool class gudusoft.gsqlparser.dlineage.util.SqlInfoHelper, which can transform dataflow coordinate to `DbObjectPosition`. + +### SqlInfoHelper + +1. First step, call api `SqlInfoHelper.getSqlInfoJson` to fetch the sqlinfo map from the DataFlowAnalyzer object, and persist it. +```java + public static String getSqlInfoJson(DataFlowAnalyzer analyzer); +``` + +2. Second step, initialize the SqlInfoHelper with the sqlinfo json string. +```java + //Constructor + public SqlInfoHelper(String sqlInfoJson); +``` + +3. Third step, transform sqlflow position string to `dataflow.model.json.Coordinate` array. + * If you use the `dataflow.model.json.DataFlow` model, you can get the Coordinate object directly, doesn't need any transform. + * If you use the `dataflow.model.xml.dataflow` model, you can call api `SqlInfoHelper.parseCoordinateString` + ```java + public static Coordinate[][] parseCoordinateString(String coordinate); + ``` + * Method parseCoordinateString support both of xml output coordinate string and json output coordinate string, like these: + ``` + //xml output coordinate string + [56,36,64e5c5241fd1311e41b2182e40f77f1e],[56,62,64e5c5241fd1311e41b2182e40f77f1e] + + //json output coordinate string + [{"x":31,"y":36,"hashCode":"64e5c5241fd1311e41b2182e40f77f1e"},{"x":31,"y":38,"hashCode":"64e5c5241fd1311e41b2182e40f77f1e"}] + ``` + +4. Fourth step, get the DbObjectPosition by api `getSelectedDbObjectInfo` +```java + public DbObjectPosition getSelectedDbObjectInfo(Coordinate start, Coordinate end); +``` + * Each position has two coordinates, start coordinate and end coordinate. If the result of DBObject.getCoordinates() has 10 items, it matches 5 positions. + * The position is based on the entire file, but not one statement. + * The sql field of DbObjectPosition return all sqls of the file. + +5. If you just want to get the specific statement information, please call the api `getSelectedDbObjectStatementInfo` +```java + public DbObjectPosition getSelectedDbObjectStatementInfo(EDbVendor vendor, Coordinate start, Coordinate end); +``` + * The position is based on the statement. + * Return the statement index of sqls, index **bases 0**. + * Return a statement, but not all sqls of the file. + +### How to use DbObjectPosition +```java +public class DbObjectPosition { + private String file; + private String filePath; + private String sql; + private int index; + private List> positions = new ArrayList>(); +} +``` +* file field matches the sql file name. +* filePath file full path. +* sql field matches the sql content. +* index: + * If the sql file is from `grabit`, it's a json file, and it has an json array named "query", the value of index field is the query item index. + * Other case, the value of index field is 0. +* positions, locations of database object, they are matched the sql field. Position x and y **base 1** but not 0. + +### Example 1 (getSelectedDbObjectInfo) +```java + String sql = "Select\n a\nfrom\n b;"; + DataFlowAnalyzer dataflow = new DataFlowAnalyzer(sql, EDbVendor.dbvmssql, false); + dataflow.generateDataFlow(new StringBuffer()); + dataflow flow = dataflow.getDataFlow(); + String coordinate = flow.getTables().get(0).getCoordinate(); + Coordinate[][] coordinates = SqlInfoHelper.parseCoordinateString(coordinate); + SqlInfoHelper helper = new SqlInfoHelper(SqlInfoHelper.getSqlInfoJson(dataflow)); + DbObjectPosition position = helper.getSelectedDbObjectInfo(coordinates[0][0], coordinates[0][1]); + System.out.println(position.getSql()); + System.out.println("table " + flow.getTables().get(0).getName() + " position is " + Arrays.toString(position.getPositions().toArray())); +``` + +Return: +```java +Select + a +from + b; + +table b position is [[4,2], [4,3]] +``` + +### Example 2 (getSelectedDbObjectStatementInfo) +```java + String sql = "Select\n a\nfrom\n b;\n Select c from d;"; + DataFlowAnalyzer dataflow = new DataFlowAnalyzer(sql, EDbVendor.dbvmssql, false); + dataflow.generateDataFlow(new StringBuffer()); + gudusoft.gsqlparser.dlineage.dataflow.model.xml.dataflow flow = dataflow.getDataFlow(); + String coordinate = flow.getTables().get(1).getCoordinate(); + Coordinate[][] coordinates = SqlInfoHelper.parseCoordinateString(coordinate); + SqlInfoHelper helper = new SqlInfoHelper(SqlInfoHelper.getSqlInfoJson(dataflow)); + DbObjectPosition position = helper.getSelectedDbObjectStatementInfo(EDbVendor.dbvmssql, coordinates[0][0], coordinates[0][1]); + System.out.println(position.getSql()); + System.out.println( + "table " + flow.getTables().get(1).getName() + " position is " + Arrays.toString(position.getPositions().toArray())); + System.out.println( + "stmt index is " + position.getIndex()); +``` + +Return: +```java + Select c from d; +table d position is [[1,20], [1,21]] +stmt index is 1 +``` + + diff --git a/linux/backend.sh b/linux/backend.sh new file mode 100644 index 0000000..7ffe95a --- /dev/null +++ b/linux/backend.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep monitor.sh|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep monitor.sh|awk '{print $2}') + kill -9 $pid +fi + +pidtotal=$(ps -ef|grep gspLive.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep gspLive.jar|awk '{print $2}') + kill -9 $pid +fi + +pidtotal=$(ps -ef|grep eureka.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep eureka.jar|awk '{print $2}') + kill -9 $pid +fi + +pidtotal=$(ps -ef|grep sqlservice.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep sqlservice.jar|awk '{print $2}') + kill -9 $pid +fi + +if [ -e $bindir/../tmp ]; then + rm -rf $bindir/../tmp +fi + +if [ -f $bindir/monitor.sh ]; then + cd $bindir + ./monitor.sh $1 $2 & +fi + +exit diff --git a/linux/eureka.sh b/linux/eureka.sh new file mode 100644 index 0000000..dad3f75 --- /dev/null +++ b/linux/eureka.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +memory=$(cat /proc/meminfo |awk '($1 == "MemTotal:"){print $2}') +if (( $memory < 8*1024*1024 )); then heapsize="256m"; else heapsize="512m"; fi + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep eureka.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep eureka.jar|awk '{print $2}') + kill -9 $pid +fi + +sleep 1 + +cd $bindir + +if [ ! -e $bindir/../log ]; then + mkdir $bindir/../log +fi + +if [ -f ../lib/eureka.jar ]; then + nohup java -Xms$heapsize -Xmx$heapsize -server -jar ../lib/eureka.jar >/dev/null 2>&1 & +else + echo $bindir/../lib/eureka.jar is not exist! +fi + +exit diff --git a/linux/gspLive.sh b/linux/gspLive.sh new file mode 100644 index 0000000..5481dee --- /dev/null +++ b/linux/gspLive.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +if [ "$1" ];then + if (( $1 == 1 )); then + gudusoft=1 + cros="--cros.allowedOrigins=https://gudusoft.com,https://www.gudusoft.com,https://sqlflow.gudusoft.com,http://sqlflow.gudusoft.com,https://api.gudusoft.com,http://gudusoft.com,http://api.gudusoft.com,http://www.gudusoft.com,http://157.230.132.40" + else + gudusoft=0 + cros="" + fi ; +else + gudusoft=0 + +fi + +if [ "$2" ];then + cros="--cros.allowedOrigins="$2 +fi + +memory=$(cat /proc/meminfo |awk '($1 == "MemTotal:"){print $2}') + +if (( $memory < 8*1024*1024 )); + then + heapsize="2g" +elif (( $memory < 16*1024*1024 )); + then + heapsize="3g" +elif (( $memory < 32*1024*1024 )); + then + heapsize="4g" +elif (( $memory < 64*1024*1024 )); + then + heapsize="8g" +else + heapsize="8g" +fi + + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep gspLive.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep gspLive.jar|awk '{print $2}') + kill -9 $pid +fi + +sleep 1 + +cd $bindir + +if [ ! -e $bindir/../log ]; then + mkdir $bindir/../log +fi + +if [ -f ../lib/gspLive.jar ]; then + nohup java -server -Xms$heapsize -Xmx$heapsize -jar ../lib/gspLive.jar $cros >/dev/null 2>&1 & +else + echo $bindir/../lib/gspLive.jar is not exist! +fi + +exit diff --git a/linux/monitor.sh b/linux/monitor.sh new file mode 100644 index 0000000..d2139e9 --- /dev/null +++ b/linux/monitor.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +bindir=$(cd `dirname $0`; pwd) +cd $bindir + +while true;do + pidtotal=$(ps -ef|grep eureka.jar|grep -v grep|wc -l) + if [ $pidtotal -eq 0 ]; then + ./eureka.sh + fi + + pidtotal=$(ps -ef|grep sqlservice.jar|grep -v grep|wc -l) + if [ $pidtotal -eq 0 ]; then + ./sqlservice.sh + fi + + pidtotal=$(ps -ef|grep gspLive.jar|grep -v grep|wc -l) + if [ $pidtotal -eq 0 ]; then + ./gspLive.sh $1 $2 + fi + + +sleep 5 +done diff --git a/linux/sqlservice.sh b/linux/sqlservice.sh new file mode 100644 index 0000000..43b5ee5 --- /dev/null +++ b/linux/sqlservice.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +memory=$(cat /proc/meminfo |awk '($1 == "MemTotal:"){print $2}') + +if (( $memory < 8*1024*1024 )); + then + heapsize="4g" +elif (( $memory < 16*1024*1024 )); + then + heapsize="10g" +elif (( $memory < 32*1024*1024 )); + then + heapsize="24g" +elif (( $memory < 64*1024*1024 )); + then + heapsize="31g" +else + heapsize="118g" +fi + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep sqlservice.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep sqlservice.jar|awk '{print $2}') + kill -9 $pid +fi + +sleep 1 + +cd $bindir +if [ ! -e $bindir/../log ]; then + mkdir $bindir/../log +fi + +if [ -f ../lib/sqlservice.jar ]; then + nohup java -server -XX:MetaspaceSize=256M -XX:MaxMetaspaceSize=256M -Xms$heapsize -Xmx$heapsize -Xss256k -XX:SurvivorRatio=8 -XX:CMSInitiatingOccupancyFraction=92 -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseConcMarkSweepGC -Djavax.accessibility.assistive_technologies=" " -jar ../lib/sqlservice.jar >/dev/null 2>&1 & +else + echo $bindir/../lib/sqlservice.jar is not exist! +fi + +exit diff --git a/linux/stop.sh b/linux/stop.sh new file mode 100644 index 0000000..99a0b76 --- /dev/null +++ b/linux/stop.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep monitor.sh|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep monitor.sh|awk '{print $2}') + kill -9 $pid + echo stop sqlflow service monitor +fi + +pidtotal=$(ps -ef|grep gspLive.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep gspLive.jar|awk '{print $2}') + kill -9 $pid + echo stop gspLive +fi + +pidtotal=$(ps -ef|grep eureka.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep eureka.jar|awk '{print $2}') + kill -9 $pid + echo stop eureka +fi + +pidtotal=$(ps -ef|grep sqlservice.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep sqlservice.jar|awk '{print $2}') + kill -9 $pid + echo stop sqlservice +fi + +exit diff --git a/mac/eureka.sh b/mac/eureka.sh new file mode 100644 index 0000000..f1340ac --- /dev/null +++ b/mac/eureka.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# less than or equal to 8G recommended +heapsize="256m"; + +# greater than 8G recommended +# heapsize="512m"; + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep eureka.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep eureka.jar|awk '{print $2}') + kill -9 $pid +fi + +sleep 1 + +cd $bindir + +if [ ! -e $bindir/../log ]; then + mkdir $bindir/../log +fi + +if [ -f ../lib/eureka.jar ]; then + nohup java -Xms$heapsize -Xmx$heapsize -server -jar ../lib/eureka.jar >/dev/null 2>&1 & +else + echo $bindir/../lib/eureka.jar is not exist! +fi + +exit diff --git a/mac/gspLive.sh b/mac/gspLive.sh new file mode 100644 index 0000000..97e2268 --- /dev/null +++ b/mac/gspLive.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# less than or equal to 8G recommended +heapsize="2g"; + +# greater than 8G recommended +# heapsize="3g"; + +if [ "$1" ];then + if (( $1 == 1 )); then + gudusoft=1 + cros="--cros.allowedOrigins=https://gudusoft.com,https://www.gudusoft.com,https://api.gudusoft.com,https://sqlflow.gudusoft.com,http://gudusoft.com,http://api.gudusoft.com,http://sqlflow.gudusoft.com,http://www.gudusoft.com,http://66.228.46.41" + else + gudusoft=0 + cros="" + fi ; +else + gudusoft=0 + +fi + +if [ "$2" ];then + cros="--cros.allowedOrigins="$2 +fi + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep gspLive.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep gspLive.jar|awk '{print $2}') + kill -9 $pid +fi + +sleep 1 + +cd $bindir + +if [ ! -e $bindir/../log ]; then + mkdir $bindir/../log +fi + +if [ -f ../lib/gspLive.jar ]; then + nohup java -server -Xms$heapsize -Xmx$heapsize -jar ../lib/gspLive.jar $cros >/dev/null 2>&1 & +else + echo $bindir/../lib/gspLive.jar is not exist! +fi + +exit diff --git a/mac/sqlservice.sh b/mac/sqlservice.sh new file mode 100644 index 0000000..38e2143 --- /dev/null +++ b/mac/sqlservice.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# less than or equal to 8G recommended +heapsize="4g"; + +# greater than 8G recommended +# heapsize="10g"; + +bindir=$(cd `dirname $0`; pwd) + +pidtotal=$(ps -ef|grep sqlservice.jar|grep -v grep|wc -l) +if [ -e $bindir -a $pidtotal -eq 1 ]; then + pid=$(ps -ef|grep -v grep|grep sqlservice.jar|awk '{print $2}') + kill -9 $pid +fi + +sleep 1 + +cd $bindir +if [ ! -e $bindir/../log ]; then + mkdir $bindir/../log +fi + +if [ -f ../lib/sqlservice.jar ]; then + nohup java -server -XX:MetaspaceSize=256M -XX:MaxMetaspaceSize=256M -Xms$heapsize -Xmx$heapsize -Xss256k -XX:SurvivorRatio=8 -XX:CMSInitiatingOccupancyFraction=92 -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseConcMarkSweepGC -Djavax.accessibility.assistive_technologies=" " -jar ../lib/sqlservice.jar >/dev/null 2>&1 & +else + echo $bindir/../lib/sqlservice.jar is not exist! +fi + +exit diff --git a/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-column-level-result.png b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-column-level-result.png new file mode 100644 index 0000000..44362b6 Binary files /dev/null and b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-column-level-result.png differ diff --git a/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-column-level.png b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-column-level.png new file mode 100644 index 0000000..257869a Binary files /dev/null and b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-column-level.png differ diff --git a/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-table-level.png b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-table-level.png new file mode 100644 index 0000000..d17b57f Binary files /dev/null and b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow-table-level.png differ diff --git a/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow.png b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow.png new file mode 100644 index 0000000..d1eecac Binary files /dev/null and b/platforms/datahub/datahub-sqlflow-dbt-customers-gudu-sqlflow.png differ diff --git a/platforms/datahub/datahub-sqlflow-dbt-customers.png b/platforms/datahub/datahub-sqlflow-dbt-customers.png new file mode 100644 index 0000000..2199185 Binary files /dev/null and b/platforms/datahub/datahub-sqlflow-dbt-customers.png differ diff --git a/platforms/datahub/datahub-sqlflow-dbt.png b/platforms/datahub/datahub-sqlflow-dbt.png new file mode 100644 index 0000000..0dff1f8 Binary files /dev/null and b/platforms/datahub/datahub-sqlflow-dbt.png differ diff --git a/platforms/datahub/readme.md b/platforms/datahub/readme.md new file mode 100644 index 0000000..c9fe286 --- /dev/null +++ b/platforms/datahub/readme.md @@ -0,0 +1,35 @@ +## Integrate Gudu SQLFlow with Datahub + +[DataHub](https://datahubproject.io/) is an open-source metadata platform for the modern data stack. + +[Gudu SQLFlow](https://sqlflow.gudusoft.com) is a powerful data lineage tool. + +Integrate the Gudu SQLFlow with the Datahub enable you easily understand the end-to-end journey of data +by tracing lineage across platforms, datasets at the column level. + +We have built a test envionment so you can see how Gudu SQLFlow works with the Datahub. + +### Demo +Please login [Datahub with Gudu SQLFlow](http://101.43.5.98:9002/) with username: datahub, password: datahub + +#### 1. After login, just click the dbt icon +datahub-dbt + +#### 2. Select the customer table +datahub-dbt + +#### 3. You will see the Gudu SQLFlow tab +datahub-dbt + +#### 4. Click the Gudu SQLFlow tab to see the upstream and downstream of this table +datahub-dbt + +#### 5. Check the column level lineage by click the lineage item +datahub-dbt + +#### 6. See the full column level lineage +datahub-dbt + + +Feel free to [contact us](https://www.gudusoft.com/contact/) if you like to integrate the Gudu SQLFlow into your datahub platform and +gain the ability to trace column-level data lineage. \ No newline at end of file diff --git a/sqlflow-search-table-column.md b/sqlflow-search-table-column.md new file mode 100644 index 0000000..3d33120 --- /dev/null +++ b/sqlflow-search-table-column.md @@ -0,0 +1,14 @@ +## Data lineage in 3 steps: Search, Select and Visualize + +Sometimes, it's not easy to find the data flow of a specific table or column +in a giant map of the enterprise's data lineage. + +Since [the SQLFlow frontend version 2.1.9](https://sqlflow.gudusoft.com), it is never so easy to find +the table and column in the schema explorer. + +Just enter the name of a table or column. [The SQLFlow](https://sqlflow.gudusoft.com) will highlight the founded +table or column automatically in the schema explorer, then select and visualize +the table or column with a right mouse click. +SQLFlow will display the data lineage of the selected table or column instantly. + +![sqlflow search table column](images/sqlflow-search-table-column.gif) \ No newline at end of file diff --git a/sqlflow-team-system.md b/sqlflow-team-system.md new file mode 100644 index 0000000..e332944 --- /dev/null +++ b/sqlflow-team-system.md @@ -0,0 +1,38 @@ +## SQLFlow Team System + +SQLFlow supports team working. All team members in an organization +can share the data lineage result for better communication. + +All members who join the team will be automatically upgraded to the premium account, +so that able to use all features of the SQLFlow. + +Members join the team don't need any additional license to use the SQLFlow, +but they need to [sign up](https://sqlflow.gudusoft.com) first to log in to the SQLFlow. + +### Create a team +When you purchase a team version license of SQLFlow. A SQLFlow team will be created +for you by the SQLFlow dev team. The email of the first team member will be +added to the team, and this member becomes the team manager of this team +who can add new member later. + +A team can includes more than one team manager. + +### Team manager + +Team manager can add member to the team. They can change the team name. +After [login the SQLFlow](https://sqlflow.gudusoft.com), click the menu icon at the right top: + +![sqlflow-team-menu](/images/sqlflow-team-menu.png) + +Add memeber: + +![sqlflow-team-manager](/images/sqlflow-team-manager.png) + +### Team quote +How many members can be added into this team. + +### Expired date +The team license will be expired after this date. + +### Team member +Team member can see jobs submitted by memebers in the same team. diff --git a/sqlflow-userid-secret.md b/sqlflow-userid-secret.md new file mode 100644 index 0000000..60c5d70 --- /dev/null +++ b/sqlflow-userid-secret.md @@ -0,0 +1,22 @@ +## SQLFlow UserId and Secret code + +In order to access the advanced features of the SQLFlow such as calling the REST API +and run the [Grabit tool](https://www.gudusoft.com/grabit/) to automated the data lineage. +You need sign-up to the https://sqlflow.gudusoft.com and +[request a 30 days premium account](https://www.gudusoft.com/request-a-premium-account/) to +get the necessary userId and secret code. + +- Once you login, please click the icon at the right top of the screen: + + + +- Click the `Account` menu item to see: + + + +Here you can +- Copy the userId +- By default, the Secret key field is empty, please click the `generate` button to create +a new secret key and copy this code. + + diff --git a/sqlflow_architecture.md b/sqlflow_architecture.md new file mode 100644 index 0000000..9d63939 --- /dev/null +++ b/sqlflow_architecture.md @@ -0,0 +1,85 @@ +## SQLFlow + +SQLFlow is a tool that automates data lineage discovery by analyzing the SQL script. +It generates a nice clean diagram to show the dataflow among the table/view and columns +in the data warehouse. + +Support more than 20 major databases, including bigquery, couchbase, dax, db2, +greenplum, hana, hive, impala, informix, mdx, mysql, netezza, openedge, oracle, postgresql, +redshift, snowflake, sqlserver, sybase, teradata, vertica, + +Just paste the SQL script and click a button, you will get the data lineage diagram instantly, +highlight the dataflow in the graph with a simple mouse click. + +You can also call the [RESTful API](https://github.com/sqlparser/sqlflow_public/tree/master/api) provided by this tool in your program and +get the data lineage and diagram model information in a JSON snippet to make further usage. + + +![SQLFlow architecture](sqlflow_architecture.png) + +### SQLFlow components +![SQLFlow components](sqlflow_components.png) + +### SQLFlow + +#### SQLFlow frontend +1. Send the SQL script received from the browser to the backend. + +2. After receiving the result, which includes the data lineage and diagram model +generated by the backend, visualize the diagram model in the browser. + +3. Highlight the dataflow in the diagram when the user clicks on a specific table/column. + +#### SQLFlow backend +1. `SQLFlow-Analyzer`: receiving the SQL script from the frontend and parse the SQL script into parse tree nodes +by utilizing [the GSP library](http://www.sqlparser.com), calculate the data lineage by analyzing AST. + + The `SQLFlow-Analyzer` component can be executed as a standalone tool. Take SQL text as input and generate data lineage in JSON format. Check [`SQLFlow.java`](https://github.com/sqlparser/gsp_demo_java/tree/master/src/main/java/demos/dlineage) for more. + +2. `FlowLayout`: Calculating the layout of database objects(table/column) in the dlineage and + generate the diagram model with all necessary position data, including nodes and edges. + `FlowLayout` depends on `doLayout library` to layout the database objects. + +3. Return a JSON snippet including the data lineage and diagram model to the frontend. + + + +### Use SQLFlow in your flavor ways + +1. Visit [SQLFlow Cloud](https://sqlflow.gudusoft.com) using the browser + + You may paste your SQL script into the SQLFlow web page or upload the SQL file to the site. + Select the correct database and then click the visualize button. + +2. Use RESTFul APIs + + SQLFlow provides RESTful API, so your program can communicate with the SQLFlow backend directly. + Sending the SQL to SQLFlow backend and receive a JSON snippet including the data lineage and diagram model for further processing in your program. + + Please note that you need to set up [the SQLFlow on-premise version](https://www.gudusoft.com/sqlflow-on-premise-version/) on your server to use the API. + + Or connect to the [SQLFlow Cloud](https://sqlflow.gudusoft.com) to use the RESTFul API + + - [SQLFlow API document](https://github.com/sqlparser/sqlflow_public/blob/master/api/readme.md) + + +3. Install both frontend and backend on your own application/server (SQLFlow on-premise version) + + Setup both the frontend and backend of SQLFlow on our server. Please check the [setup manual](install_sqlflow.md). + + To setup SQLFlow on your server, please [contact us](https://www.gudusoft.com/request-a-quote/) to obtain a commercial license to get all those distribution files + of the SQLFlow on-premise version. + + +4. Integrate the frontend and backend to your data platform + + + The front end is written in Typescript to integrate it into your frontend if you use the same tech. + + The backend part can be provided as Java library so you can be embedded into your program. + + +### Relations generated by SQLFlow +[This article](dbobjects_relationship.md) describes the relationship generated by the SQLFlow between column and column, column and table/view. +One relation includes one target column and a relationship type, and one or more source columns. + diff --git a/sqlflow_architecture.png b/sqlflow_architecture.png new file mode 100644 index 0000000..c39d829 Binary files /dev/null and b/sqlflow_architecture.png differ diff --git a/sqlflow_components.md b/sqlflow_components.md new file mode 100644 index 0000000..f012244 --- /dev/null +++ b/sqlflow_components.md @@ -0,0 +1,33 @@ +## SQLFlow components + +sqlfow was comprised of two parts: frontend and backend. + +### Logical parts +![SQLFlow components](sqlflow_components.png) + +#### SQLFlow frontend +1. Send the SQL script received from the browser in JSON to the backend. + +2. After receiving the result which includes the data lineage and diagram model +generated by the backend, visualize the diagram model in the browser. + +3. Highlight the dataflow in the diagram when the user clicks on a specific column. + +#### SQLFlow backend +1. `SQLFlow`: receiving the SQL script from the frontend and parse the SQL script into parse tree nodes +by utilizing [the GSP library](http://www.sqlparser.com), then calculate the dlineage by analyzing AST. + +2. `FlowLayout`: Calculating the layout of database objects(table/column) in the dlineage and + generate the diagram model with all necessary position data, including nodes and edges. + `FlowLayout` depends on `doLayout library` to layout the database objects. + +3. Return a JSON snippet including the data lineage and diagram model to the frontend. + + +### Physical parts + + +### Install SQLFlow + +sqlfow was comprised of two parts: frontend and backend. The frontend and backend +can be installed on the same server, or they can be installed seperated on two different servers. diff --git a/sqlflow_components.png b/sqlflow_components.png new file mode 100644 index 0000000..a6cad89 Binary files /dev/null and b/sqlflow_components.png differ diff --git a/sqlflow_faq.md b/sqlflow_faq.md new file mode 100644 index 0000000..7d22148 --- /dev/null +++ b/sqlflow_faq.md @@ -0,0 +1,8 @@ +## SQLFlow FAQ + +Q. How to clear the cache of SQLFlow backend? + +A: Remove files under those 2 directories: + +`/wings/sqlflow/backend/data/session`, +`/wings/sqlflow/backend/tmp` diff --git a/sqlflow_guide.md b/sqlflow_guide.md new file mode 100644 index 0000000..4a23980 --- /dev/null +++ b/sqlflow_guide.md @@ -0,0 +1,164 @@ +- [SQLFlow frontend guide](#sqlflow-frontend-guide) + * [video](#video) + + [1. editor](#1-editor) + - [1.1 dbvendor](#11-dbvendor) + - [1.2 sample sql](#12-sample-sql) + - [1.3 upload](#13-upload) + - [1.4 visualize](#14-visualize) + - [1.5 visualize join](#15-visualize-join) + - [1.6 login](#16-login) + + [2. schema](#2-schema) + - [seach schema](#seach-schema) + - [seach schema path](#seach-schema-path) + - [collapse all](#collapse-all) + + [3. setting](#3-setting) + + [4. job](#4-job) + + [5. download](#5-download) + + [6. sqlflow diagram panel](#6-sqlflow-diagram-panel) + + [How it works](#how-it-works) + + [Restful API](#restful-api) + + + +# SQLFlow frontend guide + +![1](https://user-images.githubusercontent.com/6293752/95873864-e2734400-0da2-11eb-85a9-e46ea43ff5c3.png) + +## video +- [SQLFlow get started](images/sqlflow_tutorial_101.gif) +- [SQLFlow visual diagram](images/sqlflow_tutorial_diagram.gif) +- [SQLFlow settings](images/sqlflow_tutorial_settings.gif) +- [SQLFlow search and visualize](images/sqlflow-tutorial-search-table-visualize.gif) + +### 1. editor + +Enter the sql into the input box, and then choose the database accordingly from the dbvendor menu, +then click visualize or visualize join button to get a clear nice data lineage diagram. + + +#### 1.1 dbvendor +Select the corresponding database. + +#### 1.2 sample sql + +Click sample sql button will load the sample SQL for the current selected database +to the input box. + +#### 1.3 upload + +upload a single SQL file, or a zip file includes multiple SQL files for processing. +A job will be created for the uploaded file. + +#### 1.4 visualize + +click visualize button will call [graph interface](#graph), and pass the following parameters： +API used when click the button. + +| name | value | +| ---------------- | ----------------------------------------- | +| sqltext | SQL query in the input box | +| dbvendor | database from the dbvendor menu | +| showRelationType | fdd | +| ignoreFunction | true | + +#### 1.5 visualize join + +click visualize join button will call [graph interface](#graph), and pass the following parameters: +API used when click the visualize join button. + +| name | value | +| ---------------- | ----------------------------------------- | +| sqltext | SQL query in the input box | +| dbvendor | database from the dbvendor menu | +| showRelationType | join | +| ignoreFunction | true | + +#### 1.6 login + +You have to login in order to upload a SQL or zip file for processing. + +### 2. schema + +show the schema objects returned by the backend. +you can select a schema/database/table and click the right mouse button to visualize the data lineage of the selected schema object. + +![3](https://user-images.githubusercontent.com/6293752/95968181-b8bc2a80-0e3f-11eb-8fc4-1501778fdc74.gif) + + +- global, green color, means all data lineage information is returned for this object. +- summary, black color, means summary information returned for this object. +- ignore record, orange color, means data lineage is returned without intermediate result. + +![image](https://user-images.githubusercontent.com/6293752/95972556-2a4aa780-0e45-11eb-8b61-2126ae9f3e0d.png) + +the color of `DATAMART、DBO` is orange, means the returned data linege doesn't include the intermediate result. +the color of `LOAN` is greeen, means all data lineage informaiton is returned. +node in gray means it's not visualized yet. + +#### seach schema + +your can search `database`,`schema`,`table`,`schema` in the schema tree. + +![动画](https://user-images.githubusercontent.com/6293752/116866667-62c59d80-ac3e-11eb-9f5f-c1e35df2f173.gif) + +#### seach schema path + +you can search `database.schema.table.column ` ,`schema.table`,`table.column` or someting else like it. When you type `dot`, the schema tree will expand the next level. + +![2](https://user-images.githubusercontent.com/6293752/116867914-69551480-ac40-11eb-9434-a581de844911.gif) + +#### collapse all + +![3](https://user-images.githubusercontent.com/6293752/116868278-14fe6480-ac41-11eb-9c9e-c7539a5b4b62.gif) + +### 3. setting + +![image](https://user-images.githubusercontent.com/6293752/95977385-6da81480-0e4b-11eb-8ec0-cc0de5466701.png) + +set the [graph interface](#graph)： + +| name | value | +| ---------------- | ------------------------------------------------------------ | +| hideColumn | false/true，hide all columns | +| showRelationType | dataflow=true, impact=false, then value is fdd.
dataflow=true, impact=true, the value is fdd,ddi,fdr,frd；
if dataflow=false, impact=true, the value is: fddi,fdr,frd； | +| ignoreRecordSet | false/true，show intermediate recordset | +| ignoreFunction | false/true，show function | + +### 4. job + +![image](https://user-images.githubusercontent.com/6293752/95977128-0b4f1400-0e4b-11eb-8c68-62657380e853.png) + +click upload button to create a job by submit SQL text file or zip file including multiple SQL files or connect to a database. + +### 5. download + +export result to json or png file for download. + +### 6. sqlflow diagram panel + +select a column to highlight the data flow, click cancel button to cancel the highlight. + +![3](https://user-images.githubusercontent.com/6293752/95986233-3ee46b00-0e58-11eb-8ee4-85a7ca5ee0f4.gif) + +right mouse click the menu item table lineage or column lineage to show the table or column relation, click cancel to back to the previous state. + +![3](https://user-images.githubusercontent.com/6293752/95986541-c336ee00-0e58-11eb-8a45-ad2d904d89ca.gif) + +### How it works + +SQLFlow frontend communicates with the backend using the RESTFul API [**/sqlflow/generation/sqlflow/graph**](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md)。 +Once it fetchs data from the backend, the frontend will analyze the graph and sqlflow and draw an interactive diagram accordingly. + +Click the button or change the value in the setting panel will result in calling different API or call the same API with the different parameters, +and get the different data from the backend consequence. + + +Reference：[SQLFlow api doc](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md) + + +### Restful API + +Reference：[SQLFlow api doc](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md) + +graph interfact： post /sqlflow/generation/sqlflow/graph + diff --git a/sqlflow_guide_cn.md b/sqlflow_guide_cn.md new file mode 100644 index 0000000..60e4384 --- /dev/null +++ b/sqlflow_guide_cn.md @@ -0,0 +1,135 @@ +# SQLFlow frontend guide + +## 原理 + +SQLFlow frontend 最主要依赖的接口是 [**/sqlflow/generation/sqlflow/graph**](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md)。从这个接口获得数据后，分析其中的 graph 和 sqlflow 字段，绘制对应的图形，并进行相关的交互。在 SQLFlow frontend 中点击不同的按钮，或者在 setting 区域做不同的设置，实质上是给这个接口传递了不同的参数，从而获得了对应的图形结果。 + +参考：[SQLFlow api 文档](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md) + +## overview + +![1660980210797](https://user-images.githubusercontent.com/6293752/185734076-58a7b974-7c5e-41ae-86ee-7bf67eb21c35.png) + +### sqltext editor + +![1](https://user-images.githubusercontent.com/6293752/185734862-10a41894-eeb8-4331-a25f-1c764ae0ebc0.gif) + +在代码编辑框输入 sql 代码，点击 dbvendor 菜单选择数据库，点击 visualize 按钮或者 visualize join 按钮，可以绘制对应的图像。 + +点击 visualize 按钮实际上是请求了[graph 接口](#graph)，并传入了下面的参数： + +| 参数 | 值 | +| ---------------- | ------------------------------------------ | +| sqltext | 代码编辑框中的代码，例如 select \* from a; | +| dbvendor | dbvendor 菜单选择的数据库，例如 dbvoracle | +| showRelationType | fdd | +| ignoreFunction | true | + +点击 visualize join 按钮实际上是请求了[graph 接口](#graph)，并传入了下面的参数： + +| 参数 | 值 | +| ---------------- | ------------------------------------------ | +| sqltext | 代码编辑框中的代码，例如 select \* from a; | +| dbvendor | dbvendor 菜单选择的数据库，例如 dbvoracle | +| showRelationType | join | +| ignoreFunction | true | + +### switch sample sql + +点击 dbvendor 菜单，选择数据库后，点击 sample sql 可以在代码编辑框中获得这个 dbvendor 对应的示例 sql，随后可以 visualize。 + +![2](https://user-images.githubusercontent.com/6293752/185735004-847cdb63-88a4-49db-8482-8820920daded.gif) + +### visualize a column or table by dropdown menu + +![11](https://user-images.githubusercontent.com/6293752/185736807-21bb3f70-3fb2-47d6-a97d-c910b139fcbc.gif) + +### hover sqltext to highlight graph + +鼠标在 sqltext 上悬停，可以在图形中找到对应的图形。 + +![3](https://user-images.githubusercontent.com/6293752/185735065-d22debe6-6dbf-417d-9e61-798b28d9ddf6.gif) + +### hover graph to highlight sqltext + +鼠标在图形上悬停，可以在 sqltext 中找到对应的代码。 + +![4](https://user-images.githubusercontent.com/6293752/185735156-de5d071a-1a55-4914-81a4-90aac85aa036.gif) + +### resize left panel width + +鼠标悬停在面板边缘悬停，如果有高亮效果，则可以拖动跳转宽度。 + +![5](https://user-images.githubusercontent.com/6293752/185735279-20b41fb1-a191-40fa-9fc1-d258246ea0fe.gif) + +### pin graph, drag graph, and cancel + +点击图形某个 column，可以固定上下游关系。长按鼠标左键，可以移动画布。 + +![6](https://user-images.githubusercontent.com/6293752/185735432-0ef385fd-b1b8-4269-ae47-e339e2b78bf5.gif) + +## setting + +设置当前[graph 接口](#graph)接口的参数，来获得不同的分析结果： + +| 参数 | 值 | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| showRelationType | 如果 direct dataflow = true，为 fdd；
如果 direct dataflow = true 且 indirect dataflow=true ，为 fdd,fddi,fdr,frd； | +| dataflowOfAggregateFunction | direct 或者 indirect，取决于 dataflowOfAggregateFunction | +| ignoreRecordSet | false 或者 true，取决于 show intermediate recordset | +| ignoreFunction | false 或者 true，取决于 show function | +| showConstantTable | false 或者 true，取决于 show constant | +| showTransform | false 或者 true，取决于 show transform | + +切换不同的选项，观察网络请求，可以看到接口参数的变化： + +![7](https://user-images.githubusercontent.com/6293752/185736267-6eefb036-f047-4a72-a95f-391847e5f145.gif) + +### show function + +![8](https://user-images.githubusercontent.com/6293752/185736347-1ce8fbf9-b66e-45e8-af75-137b746bc31d.gif) + +### show transform + +![10](https://user-images.githubusercontent.com/6293752/185736610-6fba47eb-9dba-42cc-9f00-5af3ad22563f.gif) + +## job list + +![image](https://user-images.githubusercontent.com/6293752/185734108-5dc282df-0b49-4061-af2d-c9fa21ab885a.png) + +### create a job + +![12](https://user-images.githubusercontent.com/6293752/185737736-814ae584-ab72-4be6-a4f6-6393607d385f.gif) + +### backwards in code + +![14](https://user-images.githubusercontent.com/6293752/185738467-b8485e3c-cbc4-4ceb-ab20-5e869908551b.gif) + +## schema + +![image](https://user-images.githubusercontent.com/6293752/185734174-6d507fb8-2cb3-4a75-a5b7-4ab70ba8addd.png) + +显示 sql 的 schema 结构。在 schema、database、table 上点击鼠标右键，可以 visualize。 + +![13](https://user-images.githubusercontent.com/6293752/185738098-7ebe1e25-816e-4178-8f60-220d02c17b00.gif) + +global、summay、ignore record 表示的是[graph 接口](#graph)返回的的三个模式（json 中的 mode 字段），分别用三种不同的颜色表示。 + +![image](https://user-images.githubusercontent.com/6293752/185738177-a7b66a2f-9532-4669-87b8-d6284d4bf03b.png) + +ANALYTICS、ENTITY 前的图标为绿色，表示 mode 为 global；DATAMART 前为黑色，表示 mode 为 summary；其他节点前为灰色，表示当前节点还没有被 visualize。 + +### view DDL + +![15](https://user-images.githubusercontent.com/6293752/185738579-962c06d1-80c0-4b61-8251-f5541d0564d3.gif) + +## export + +导出处理结果为 json 或者 png: +![1660980656500](https://user-images.githubusercontent.com/6293752/185734305-70c24757-c59c-40b4-b235-a79a214b7472.png) + +## 接口 + +参考：[SQLFlow api 文档](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md) + +graph 接口： post /sqlflow/generation/sqlflow/graph diff --git a/sqlflow_introduce.md b/sqlflow_introduce.md new file mode 100644 index 0000000..04b9a83 --- /dev/null +++ b/sqlflow_introduce.md @@ -0,0 +1,37 @@ +## SQLFlow Introduce + +The SQLFlow is a tool helps you collect data lineage information by analying the SQL scripts +in a governed data environment. It scan SQL code to understand all the logic and reverse engineer it, +to build an understanding of how data changes and which data serves as input for calculating other data. + +Once the metadata of the data lineage is ready, SQLFlow presents a nice clean graph to you that tells +where the data came from, what transformations it underwent along the way, +and what other data items are derived from this data value. + +![SQLFlow Introduce](images/sqlflow_introduce1.png) + +### What SQLFlow can do for you +- Provide a nice cleam diagram to the end-user to understand the data lineage quickly. +- Incorporate the lineage metadata decoded from the complex SQL script into your own metadata database for further processing. +- Visualize the metadata already existing in your database to release the power of data. +- Perform impact analysis and root-cause analysis by tracing lineage backwards or forwards with several mouse click. +- Able to process SQL script from more than 20 major database vendors. + +### How to use SQLFlow +- Open the official website of the SQLFlow and paste your SQL script or metadata to get a nice clean lineage diagram. +- Call the Restful API of the SQLFlow to get data lineage metadata decoded by the SQLFlow from the SQL script. +- The on-premise version of SQLflow enables you to use it on your own server to keep the data safer. + + +### The price plan +- [SQLFlow price plan](sqlflow_pricing_plans.md) + +### Restful APIs +- [SQLFlow API document](https://github.com/sqlparser/sqlflow_public/blob/master/api/sqlflow_api.md) +- [Client in C#](https://github.com/sqlparser/sqlflow_public/tree/master/api/client/csharp) + +### SQLFlow architecture +- [Architecture document](sqlflow_architecture.md) + +### User manual +- [Working in progress]() diff --git a/sqlflow_privacy-policy.md b/sqlflow_privacy-policy.md new file mode 100644 index 0000000..8137a10 --- /dev/null +++ b/sqlflow_privacy-policy.md @@ -0,0 +1,85 @@ +### PRIVACY POLICY & COOKIES + +#### 1. What information do we collect? + +We may collect, store and use the following kinds of personal information: +- Information about your computer and about your visits to and use of this website (including your IP address, geographical location, browser type and version, operating system, referral source, length of visit, page views and website navigation). +- Information that you provide us for the purpose of subscribing to our website services, email notifications and/or newsletters. +- Any other information such as SQL code that you choose to send us. + + +#### 2. Privacy and Data Protection +When you use SQLFlow your information is protected by this Privacy Policy. +Information provided to the Service provider will under no circumstances be shared, +passed, disclosed, rented or sold to a third party unless required by law or other regulation. +On rare occasions the Service provider may need to contact the Customer with essential messages regarding the service. + +Any data handled by SQLFlow is under the protection of this Data Protection Policy. +The Service provider has strict internal procedures designed to protect the Customer’s personal information +and data from unauthorised access, improper use, alteration, unintended destruction or other loss. + +- All data in transit are transmitted over SSL with a signed certificate +- The Service runs on privately hosted servers in certified data centers. We guarantee the location to be within US. + +#### 3. Cookies + +A cookie consists of information sent by a web server to a web browser, and it is stored by the browser. The information is then sent back to the server each time the browser requests a page from the server. This enables the web server to identify and track the web browser. + +We may use both “session” cookies and “persistent” cookies on the website. We will use the session cookies to keep track of you whilst you navigate the website and persistent cookies to enable our website to recognize you when you visit. Session cookies will be deleted from your computer when you close your browser. Persistent cookies will remain stored on your computer until deleted, or until they reach a specified expiration date. + +We use Google Analytics to analyze the use of this website. Google Analytics generates statistical and other information about website use by means of cookies, which are stored on users’ computers. The information generated regarding our website is used to create reports about the use of the website. Google will store this information. Google’s privacy policy is available at: http://www.google.com/privacypolicy.html + +Most browsers allow you to reject all cookies, while some browsers allow you to reject just third party cookies. Blocking all cookies will, however, have a negative impact on the usability of many websites, and we cannot guarantee you the best website performance possible. + + +#### 4. Using your personal information + +Personal information submitted to us via this website will be used for the purposes specified in this privacy policy or in relevant parts of the website. + +We may use your personal information to: +- improve your browsing experience by personalizing the website; +- enable your use of the services available on the website; +- send you email notifications which you have specifically requested; +- send you marketing materials related to our business by email (you can inform us at any time if you no longer wish to receive marketing email); +- provide third parties with statistical information about our users – but this information will not be used to identify any individual user; +- pass on to third parties with whom we have signed a Reseller Agreement/Support Provider Agreement solely for the purposes of negotiating a business deal with you or providing you with support and/or maintenance for our product(s) + +If you submit personal information for publication on our website, we will publish and otherwise use that information in accordance with the license you grant us. We will not provide your personal information to any third parties for the purpose of direct marketing + + +#### 5. Disclosures + +We may disclose information about you to our employees, officers, agents, suppliers and subcontractors insofar as is reasonably necessary for the purposes as set forth in this privacy policy. + +In addition, we may disclose your personal information: +- to the extent that we are required to do so by law; +- in connection with any legal proceedings or prospective legal proceedings; +- in order to establish, exercise or defend our legal rights (including providing information to others for the purposes of fraud prevention and reducing credit risk). + +Except as provided for in this privacy policy, we will not provide your information to third parties. + +#### 6. Security of your personal information + +We will take reasonable technical to prevent the loss, misuse or alteration of your personal information. All electronic transactions made to or from us will be encrypted using SSL technology. Of course, data transmission over the internet is inherently insecure, and we cannot guarantee the security of data sent over the internet. + +You are responsible for keeping your password and user details confidential. We will not ask you for your password (except when you log in to the website). + +#### 7. Policy amendments + +We may update this privacy policy from time to time by posting a new version on our website. You should check this page occasionally to ensure you are happy with any changes. We may also notify you of changes to our privacy policy by email. + +#### 8. Your rights + +You may instruct us to provide you with any personal information we hold about you. We may withhold such personal information to the extent permitted by law. + +You may instruct us not to process your personal information for marketing purposes by email at any time. In practice, we will provide you with an opportunity to opt-out of the use of your personal information for marketing purposes. + +#### 9. Third party websites + +The website contains links to other websites. We are not responsible for the privacy policies or practices of third party websites. + +#### 10. Updating information + +Please let us know if the personal information which we hold about you needs to be corrected or updated. + +If you have any questions about this privacy policy or how we handle your personal information, please email us at support@gudusoft.com. \ No newline at end of file diff --git a/sqlflow_terms.md b/sqlflow_terms.md new file mode 100644 index 0000000..a64bbff --- /dev/null +++ b/sqlflow_terms.md @@ -0,0 +1,134 @@ +### TERMS & CONDITIONS + +This document outlines the terms and conditions under which Gudu Software (“Licensor”), provides license to access its website/service to visitors (“you”). +For the purposes of this Terms & Conditions, “website” is a set of pages located under domain names "gudusoft.com" and its subdomains. + +These terms and conditions govern your use of this website/service, by using this website/service, you accept these terms and conditions in full. +If you disagree with these terms and conditions or any part of these terms and conditions, you must not use this website. + +The Licensor may revise these terms and conditions from time-to-time. Revised terms and conditions +will apply to the use of this website from the date of the revised terms and conditions’ publication on this website. +Please check this page regularly to ensure you are familiar with the current version. + +#### Description of Service + +SQLFlow is a cloud service which generates data-lineage and an interactive diagram showing data flows and dependencies +of SQL code from their existing data warehouses and relational databases. + +Data professionals benefit from transformation of SQL code into data-lineage maps as it replaces the need to perform manual code analysis. + +The SQLFlow service can be accessed via web browser or REST API or setup on your own server. The user submits existing SQL code to SQLFlow, +SQLFlow processes the SQL code and layout an interactive diagram of data flows with added productivity features. + +#### License to use the website/Service + +SQLFlow is the intellectual property of the Service provider, accessible as a cloud service for the duration of the license provided to +the Customer on a subscription basis. The license to use SQLFlow is non-transferable, and non-exclusive. +The license is for named users only, and the sharing of accounts is forbidden. + +Code submitted by the Customer must be the property of the Customer. If the Customer wishes to submit code owned by a third party, +a further agreement must be made with the Service provider to allow this. + +By uploading code to the Service provider, the customer retains full rights and ownership of that code, +and gives the Service provider license to securely process that code in order to provide the Service. + +The Service provider has the ownership of computed metadata but hereby grants exclusive rights to Customer: + +- to freely access, modify or otherwise manipulate such metadata. Metadata can be exported at any given time through the REST API and further processed without any limitations. + +- to link to or incorporate aspects of the visualization page into any information system belonging to the Customer (internal wiki, etc.) for internal purposes only. e.g. +The Customer is entitled to use or integrate the product of the SQLFlow service into their own infrastructure. + + +Unless otherwise stated, the Licensor owns the intellectual property rights to the website and all material on the website. +Subject to the license below, all these intellectual property rights are reserved. + +You may view, download for caching purposes only, and print pages or any other content from the website, +subject to the restrictions set out in these terms and conditions. + +You must not: +- republish material from this website (including republication on another website); +- sell, rent or sub-license material from the website; +- reproduce, duplicate, copy or otherwise exploit material on this website for a commercial purpose; +- edit or otherwise modify any material on the website; +- redistribute material from this website, except for content specifically and expressly made available for redistribution. + +#### Accessing SQLFlow +To use SQLFlow, the Customer must agree to these terms. It's free to use the SQLFlow service via web browser. + +The cusotmer need to pay subscription fee in order to use RESTFul API of SQLFlow Service. +The Customer must specify the amount of users and intended duration of use. Gudu software will provide an invoice, and upon payment, +the Service will be available to the Customer for the specified period. +For the purposes of a free trial, the same conditions apply, with the exception that the Customer will not receive an invoice unless a paid service is explicitly requested. + +To continue subscription to SQLFlow, the Customer must inform the Service provider no later than 10 days before the end of the current subscription. +Access to the Service will continue following the end of the initial subscription period, providing that the subscription fee for the following period is paid. +If the Customer wishes to upgrade subscription tier, the difference in subscription costs must be paid in advance. + +The Customer has the right to cancel the Service if the service provider breaks any of these terms. +In such a case, the equivalent fee for the remaining duration of subscription will be refunded. +Otherwise cancellation of the service will not entitle the Customer to a refund for the subscribed period. + +#### Setup SQLFlow on your own server +The cusotmer need to pay a one time license fee in order to setup SQLFlow on their own server. +Customer may install the SQLFlow only on one server. +No more than ten IP address allowed to access the RESTFul API. + + +#### Acceptable use + +You must not use this website in any way that causes, or may cause, damage to the website or the impairment of the availability or accessibility of the website; or in any way which is unlawful, illegal, fraudulent, or harmful; or in connection with any unlawful, illegal, fraudulent, or harmful purpose or activity. + +You must not use this website to copy, store, host, transmit, send, use, publish or distribute any material which consists of (or is linked to) any spyware, computer virus, Trojan horse, worm, keystroke logger, rootkit or other malicious computer software. + +You must not conduct any systematic or automated data collection activities (including without limitation scraping, data mining, data extraction and data harvesting) on or in relation to this website without the Licensor’s express written consent. + + +#### User content + +In these terms and conditions, “your user content” means material (including without limitation text and images) that you submit to this website, for whatever purpose. + +You grant the Licensor a worldwide, irrevocable, non-exclusive, royalty-free license to use, reproduce, adapt, publish, translate and distribute your user content in any existing or future media. You also grant the Licensor the right to sub-license these rights, and the right to take action for the infringement of these rights. + +Your user content must not be illegal or unlawful, must not infringe on any third party’s legal rights, and must not be capable of giving rise to legal action whether against you or the Licensor or a third party (in each case under any applicable law). + +The Licensor reserves the right to edit or remove any material submitted to this website, or stored on the Licensor’s servers, or hosted or published upon this website. + +#### No warranties + +This website is provided “as is” without any representations or warranties, express or implied. The Licensor makes no representations or warranties in relation to this website or the information and materials provided on this website. + +Without prejudice to the generality of the foregoing paragraph, the Licensor does not warrant that: +- this website will be constantly available, or available at all; or +- the information on this website is complete, true, accurate or non-misleading. + +Nothing on this website constitutes, or is meant to constitute, advice of any kind. + +#### Limitations of liability + +The Licensor will not be liable to you in any way in relation to the contents of, or use of, or otherwise in connection with this website: +- to the extent that the website is provided free-of-charge, for any direct loss; +- for any indirect, special or consequential loss; or +- for any business losses, loss of revenue, income, profits or anticipated savings, loss of contracts or business relationships, loss of reputation or goodwill, or loss or corruption of information or data. + +These limitations of liability apply even if the Licensor has been expressly advised of the potential loss. + + +#### Indemnity + +You hereby indemnify the Licensor and undertake to keep the Licensor indemnified against any losses, damages, costs, liabilities and expenses (including without limitation legal expenses and any amounts paid by the Licensor to a third party in the settlement of a claim or dispute on the advice of the Licensor’s legal advisers) incurred or suffered by the Licensor arising out of any breach by you of any provision of these terms and conditions, or arising out of any claim that you have breached any provision of these terms and conditions. + +#### Breaches of these terms and conditions + +Without prejudice to the Licensor’s other rights under these terms and conditions, if you breach these terms and conditions in any way, the Licensor may take such action as the Licensor deems appropriate to deal with the breach, including suspending your access to the website, prohibiting you from accessing the website, blocking computers using your IP address from accessing the website, contacting your internet service provider to request that they block your access to the website and/or bringing court proceedings against you. + +#### Assignment + +The Licensor may transfer, sub-contract or otherwise deal with the Licensor’s rights and/or obligations under these terms and conditions without notifying you or obtaining your consent. You may not transfer, sub-contract or otherwise deal with your rights and/or obligations under these terms and conditions. + + +#### Law and jurisdiction + +These terms and conditions will be governed by and construed in accordance with the laws of the Hong Kong, +and any disputes relating to these terms and conditions will be subject to the exclusive jurisdiction of the courts of the Hong Kong. +If you have any questions about this privacy policy or how we handle your personal information, please email us at support@gudusoft.com \ No newline at end of file diff --git a/sqlflowjs/config.private.json b/sqlflowjs/config.private.json new file mode 100644 index 0000000..df99737 --- /dev/null +++ b/sqlflowjs/config.private.json @@ -0,0 +1,45 @@ +{ + "SqlTextLengthMax": 307200, + "ApiPrefix": "", + "Authorization": { + "base64token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYwMzc1NjgwMCwiaWF0IjoxNTcyMjIwODAwfQ.EhlnJO7oqAHdr0_bunhtrN-TgaGbARKvTh2URTxu9iU" + }, + "Toolbar": { + "built-in": [ + "SQL", + "Schema Explorer", + "Setting", + "Export", + "Version", + "Share" + ], + "extra": [] + }, + "EnableAuth0": false, + "Auth0": { + "domain": "gudusoft.auth0.com", + "clientID": "PZ7fv0vKLL8f0HFW0FIlP1PiDeF0h5HD" + }, + "SampleSql": { + "bigquery": "MERGE dataset.DetailedInventory T\nUSING dataset.Inventory S\nON T.product = S.product\nWHEN NOT MATCHED AND quantity < 20 THEN\n INSERT(product, quantity, supply_constrained, comments)\n VALUES(product, quantity, true, ARRAY>[(DATE('2016-01-01'), 'comment1')])\nWHEN NOT MATCHED THEN\n INSERT(product, quantity, supply_constrained)\n VALUES(product, quantity, false)\n;", + "couchbase": "-- Couchbase\nSELECT t1.country, array_agg(t1.city), sum(t1.city_cnt) as apnum\nFROM (SELECT city, city_cnt, array_agg(airportname) as apnames, country\n FROM `travel-sample` WHERE type = \"airport\"\n GROUP BY city, country LETTING city_cnt = count(city) ) AS t1\nWHERE t1.city_cnt > 5\nGROUP BY t1.country;\n\nINSERT INTO `travel-sample` (key UUID(), value _country)\n SELECT _country FROM `travel-sample` _country\n WHERE type = \"airport\" AND airportname = \"Heathrow\";\n\t \nMERGE INTO all_empts a USING emps_deptb b ON KEY b.empId\nWHEN MATCHED THEN\n UPDATE SET a.depts = a.depts + 1,\n a.title = b.title || \", \" || b.title\nWHEN NOT MATCHED THEN\n INSERT { \"name\": b.name, \"title\": b.title, \"depts\": b.depts, \"empId\": b.empId, \"dob\": b.dob }\n;\n\nUPDATE `travel-sample`\nSET foo = 9\nWHERE city = (SELECT raw city FROM `beer-sample` WHERE type = \"brewery\")\n;", + "db2": "-- DB2\nSELECT PART, SUPPLIER, PRODNUM, PRODUCT\nFROM (SELECT PART, PROD# AS PRODNUM, SUPPLIER\nFROM PARTS\nWHERE PROD# < 200) AS PARTX\nLEFT OUTER JOIN PRODUCTS\nON PRODNUM = PROD#;\n\n\nSELECT D.DEPTNO, D.DEPTNAME,\nEMPINFO.AVGSAL, EMPINFO.EMPCOUNT\nFROM DEPT D,\nTABLE (SELECT AVG(E.SALARY) AS AVGSAL,\nCOUNT(*) AS EMPCOUNT\nFROM EMP E\nWHERE E.WORKDEPT = D.DEPTNO)\nAS EMPINFO;\n\nDELETE FROM EMP X\nWHERE ABSENT = (SELECT MAX(ABSENT) FROM EMP Y\nWHERE X.WORKDEPT = Y.WORKDEPT);\n\nINSERT INTO SMITH.TEMPEMPL\nSELECT *\nFROM DSN8B10.EMP;\n\nINSERT INTO B.EMP_PHOTO_RESUME\n-- OVERRIDING USER VALUE\nSELECT * FROM DSN8B10.EMP_PHOTO_RESUME;\n\nMERGE INTO RECORDS AR\nUSING (VALUES (:hv_activity, :hv_description)\n--FOR :hv_nrows ROWS\n)\nAS AC (ACTIVITY, DESCRIPTION)\nON (AR.ACTIVITY = AC.ACTIVITY)\nWHEN MATCHED THEN UPDATE SET DESCRIPTION = AC.DESCRIPTION\nWHEN NOT MATCHED THEN INSERT (ACTIVITY, DESCRIPTION)\nVALUES (AC.ACTIVITY, AC.DESCRIPTION)\n-- NOT ATOMIC CONTINUE ON SQLEXCEPTION\n;\n\nCREATE TABLE T1 (COL1 CHAR(7), COL2 INT);\nINSERT INTO T1 VALUES ('abc', 10);\nMERGE INTO T1 AS A\nUSING TABLE (VALUES ('rsk', 3 ) ) AS T (ID, AMOUNT)\nON A.COL1 = T.ID\nWHEN MATCHED\nTHEN UPDATE SET COL2 = CARDINALITY(CHARA)\nWHEN NOT MATCHED\nTHEN INSERT (COL1, COL2 ) VALUES (T.ID, CARDINALITY(INTA));\n\nUPDATE DSN8B10.EMP\nSET PROJSIZE = (SELECT COUNT(*)\nFROM DSN8B10.PROJ\nWHERE DEPTNO = 'E21')\nWHERE WORKDEPT = 'E21';\n\nCREATE VIEW DSN8B10.FIRSTQTR (SNO, CHARGES, DATE) AS\nSELECT SNO, CHARGES, DATE\nFROM MONTH1\nWHERE DATE BETWEEN '01/01/2000' and '01/31/2000'\nUNION All\nSELECT SNO, CHARGES, DATE\nFROM MONTH2\nWHERE DATE BETWEEN '02/01/2000' and '02/29/2000'\nUNION All\nSELECT SNO, CHARGES, DATE\nFROM MONTH3\nWHERE DATE BETWEEN '03/01/2000' and '03/31/2000';\n\n/* not supported yet\n\nEXEC SQL\nDECLARE C1 CURSOR FOR\nSELECT BONUS\nFROM DSN8710.EMP\nWHERE WORKDEPT = 'E12'\nFOR UPDATE OF BONUS;\n\nEXEC SQL\nUPDATE DSN8710.EMP\nSET BONUS = ( SELECT .10 * SALARY FROM DSN8710.EMP Y\nWHERE EMPNO = Y.EMPNO )\nWHERE CURRENT OF C1;\n*/", + "greenplum": "-- greenplum sample SQL\n\nWITH regional_sales AS (\n SELECT region, SUM(amount) AS total_sales\n FROM orders\n GROUP BY region\n ), top_regions AS (\n SELECT region\n FROM regional_sales\n WHERE total_sales > (SELECT SUM(total_sales)/10 FROM regional_sales)\n )\nSELECT region,\n product,\n SUM(quantity) AS product_units,\n SUM(amount) AS product_sales\nFROM orders\nWHERE region IN (SELECT region FROM top_regions)\nGROUP BY region, product;\n\nWITH RECURSIVE search_graph(id, link, data, depth, path, cycle) AS (\n SELECT g.id, g.link, g.data, 1,\n ARRAY[g.id],\n false\n FROM graph g\n UNION ALL\n SELECT g.id, g.link, g.data, sg.depth + 1,\n path || g.id,\n g.id = ANY(path)\n FROM graph g, search_graph sg\n WHERE g.id = sg.link AND NOT cycle\n)\nSELECT * FROM search_graph;\n\nINSERT INTO films SELECT * FROM tmp_films WHERE date_prod < '2004-05-07';\n\nUPDATE employees SET sales_count = sales_count + 1 WHERE id =\n (SELECT id FROM accounts WHERE name = 'Acme Corporation');\n \n \nUPDATE accounts SET (contact_last_name, contact_first_name) =\n (SELECT last_name, first_name FROM salesmen\n WHERE salesmen.id = accounts.sales_id); ", + "hana": "-- sap hana sample SQLCODE\n\nCREATE TABLE t1(a INT, b INT, c INT); \nCREATE TABLE t2(a INT, b INT, c INT);\nINSERT INTO t1 VALUES(1,12,13); \nINSERT INTO t2 VALUES(2,120,130);\n \nINSERT INTO t1 WITH alias1 AS (SELECT * FROM t1) SELECT * FROM alias1;\nINSERT INTO t1 WITH w1 AS (SELECT * FROM t2) SELECT * FROM w1;\n\nSELECT TA.a1, TB.b1 FROM TA, LATERAL (SELECT b1, b2 FROM TB WHERE b3 = TA.a3) TB WHERE TA.a2 = TB.b2;\n\nUPDATE T1 SET C1 = ARRAY( SELECT C1 FROM T0 ) WHERE ID = 1;\n\nUPSERT T1 VALUES ( 1, ARRAY ( SELECT C1 FROM T0 ) ) WHERE ID = 1;\n\n\nMERGE INTO \"my_schema\".t1 USING \"my_schema\".t2 ON \"my_schema\".t1.a = \"my_schema\".t2.a\n WHEN MATCHED THEN UPDATE SET \"my_schema\".t1.b = \"my_schema\".t2.b\n WHEN NOT MATCHED THEN INSERT VALUES(\"my_schema\".t2.a, \"my_schema\".t2.b);\n \n MERGE INTO T1 USING T2 ON T1.A = T2.A\n WHEN MATCHED AND T1.A > 1 THEN UPDATE SET B = T2.B\n WHEN NOT MATCHED AND T2.A > 3 THEN INSERT VALUES (T2.A, T2.B);\n \n \n /* not support yet\n CREATE VIEW Illness_K_Anon (ID, Gender, Location, Illness)\n AS SELECT ID, Gender, City AS Location, Illness\n FROM Illness\n WITH ANONYMIZATION ( ALGORITHM 'K-ANONYMITY'\n PARAMETERS '{\"data_change_strategy\": \"qualified\", \"k\": 2}'\n COLUMN ID PARAMETERS '{\"is_sequence\": true}'\n COLUMN Gender PARAMETERS '{\"is_quasi_identifier\":true, \"hierarchy\":{\"embedded\": [[\"F\"], [\"M\"]]}}'\n COLUMN Location PARAMETERS '{\"is_quasi_identifier\":true, \"hierarchy\":{\"embedded\": [[\"Paris\", \"France\"], [\"Munich\", \"Germany\"], [\"Nice\", \"France\"]]}}');\n*/", + "hive": "-- Hive sample sql\nSELECT t3.col\nFROM (\n SELECT a+b AS col\n FROM t1\n UNION ALL\n SELECT c+d AS col\n FROM t2\n) t3\n;\n\nSELECT a.val, b.val, c.val FROM a JOIN b ON (a.key = b.key1) JOIN c ON (c.key = b.key2)\n;\n\nCREATE DATABASE merge_data;\n \nCREATE TABLE merge_data.transactions(\n\tID int,\n\tTranValue string,\n\tlast_update_user string)\nPARTITIONED BY (tran_date string)\nCLUSTERED BY (ID) into 5 buckets \nSTORED AS ORC TBLPROPERTIES ('transactional'='true');\n \nCREATE TABLE merge_data.merge_source(\n\tID int,\n\tTranValue string,\n\ttran_date string)\nSTORED AS ORC;\n\n-- merge is not supported yet.\n\n-- MERGE INTO merge_data.transactions AS T \n-- USING merge_data.merge_source AS S\n-- ON T.ID = S.ID and T.tran_date = S.tran_date\n-- WHEN MATCHED AND (T.TranValue != S.TranValue AND S.TranValue IS NOT NULL) THEN UPDATE SET TranValue = S.TranValue, last_update_user = 'merge_update'\n-- WHEN MATCHED AND S.TranValue IS NULL THEN DELETE\n-- WHEN NOT MATCHED THEN INSERT VALUES (S.ID, S.TranValue, 'merge_insert', S.tran_date)", + "impala": "-- Impala sample sql\ninsert into t2 select * from t1;\ninsert into t2 select c1, c2 from t1;\n\nCREATE VIEW v5 AS SELECT c1, CAST(c3 AS STRING) c3, CONCAT(c4,c5) c5, TRIM(c6) c6, \"Constant\" c8 FROM t1;\n\nCREATE VIEW v7 (c1 COMMENT 'Comment for c1', c2) COMMENT 'Comment for v7' AS SELECT t1.c1, t1.c2 FROM t1;", + "informix": "-- Informix sample sql\nCREATE VIEW myview (cola, colb) AS\nSELECT colx, coly from firsttab\nUNION\nSELECT colx, colz from secondtab;\n\nCREATE VIEW palo_alto AS\nSELECT * FROM customer WHERE city = 'Palo Alto'\nWITH CHECK OPTION\n;\n\nMERGE INTO t2 AS o USING t1 AS n ON o.f1 = n.f1\nWHEN NOT MATCHED THEN INSERT ( o.f1,o.f2)\nVALUES ( n.f1,n.f2)\n;\n\nINSERT INTO t2(f1, f2)\nSELECT t1.f1, t1.f2 FROM t1\nWHERE NOT EXISTS\n(SELECT f1, f2 FROM t2\nWHERE t2.f1 = t1.f1);\n\nMERGE INTO sale USING new_sale AS n\nON sale.cust_id = n.cust_id\nWHEN MATCHED THEN UPDATE\nSET sale.salecount = sale.salecount + n.salecount\nWHEN NOT MATCHED THEN INSERT (cust_id, salecount)\nVALUES (n.cust_id, n.salecount);\n\n\nMERGE INTO customer c\nUSING ext_customer e\nON c.customer_num=e.customer_num AND c.fname=e.fname AND c.lname=e.lname\nWHEN NOT MATCHED THEN\nINSERT\n(c.fname, c.lname, c.company, c.address1, c.address2,\nc.city, c.state, c.zipcode, c.phone)\nVALUES\n(e.fname, e.lname, e.company, e.address1, e.address2,\ne.city, e.state, e.zipcode, e.phone)\nWHEN MATCHED THEN UPDATE\nSET c.fname = e.fname,\nc.lname = e.lname,\nc.company = e.company,\nc.address1 = e.address1,\nc.address2 = e.address2,\nc.city = e.city,\nc.state = e.state,\nc.zipcode = e.zipcode,\nc.phone = e.phone ;\n\n\nUPDATE nmosdb@wnmserver1:test\nSET name=(SELECT name FROM test\nWHERE test.id = nmosdb@wnmserver1:test.id)\nWHERE EXISTS(\nSELECT 1 FROM test WHERE test.id = nmosdb@wnmserver1:test.id\n);\n\nUPDATE orders\nSET ship_charge =\n(SELECT SUM(total_price) * .07 FROM items\nWHERE orders.order_num = items.order_num)\nWHERE orders.order_num = 1001;", + "mdx": "mdx", + "mysql": "-- mysql sample sql\n\nSELECT\n salesperson.name,\n -- find maximum sale size for this salesperson\n (SELECT MAX(amount) AS amount\n FROM all_sales\n WHERE all_sales.salesperson_id = salesperson.id)\n AS amount,\n -- find customer for this maximum size\n (SELECT customer_name\n FROM all_sales\n WHERE all_sales.salesperson_id = salesperson.id\n AND all_sales.amount =\n -- find maximum size, again\n (SELECT MAX(amount) AS amount\n FROM all_sales\n WHERE all_sales.salesperson_id = salesperson.id))\n AS customer_name\nFROM\n salesperson;\n \n \nSELECT\n salesperson.name,\n max_sale.amount,\n max_sale_customer.customer_name\nFROM\n salesperson,\n -- calculate maximum size, cache it in transient derived table max_sale\n (SELECT MAX(amount) AS amount\n FROM all_sales\n WHERE all_sales.salesperson_id = salesperson.id)\n AS max_sale,\n -- find customer, reusing cached maximum size\n (SELECT customer_name\n FROM all_sales\n WHERE all_sales.salesperson_id = salesperson.id\n AND all_sales.amount =\n -- the cached maximum size\n max_sale.amount)\n AS max_sale_customer;\n\nSELECT\n salesperson.name,\n max_sale.amount,\n max_sale.customer_name\nFROM\n salesperson,\n -- find maximum size and customer at same time\n LATERAL\n (SELECT amount, customer_name\n FROM all_sales\n WHERE all_sales.salesperson_id = salesperson.id\n ORDER BY amount DESC LIMIT 1)\n AS max_sale;\n\n/* syntax not supported yet \nWITH RECURSIVE employee_paths (id, name, path) AS\n(\n SELECT id, name, CAST(id AS CHAR(200))\n FROM employees\n WHERE manager_id IS NULL\n UNION ALL\n SELECT e.id, e.name, CONCAT(ep.path, ',', e.id)\n FROM employee_paths AS ep JOIN employees AS e\n ON ep.id = e.manager_id\n)\nSELECT * FROM employee_paths ORDER BY path;\n*/\n\nUPDATE table1 t1 \nJOIN table2 t2 ON t1.field1 = t2.field1 \nJOIN table3 t3 ON (t3.field1=t2.field2 AND t3.field3 IS NOT NULL) \nSET t1.field9=t3.field9\nWHERE t1.field5=1\nAND t1.field9 IS NULL ", + "netezza": "-- netezza sample sql\n\nINSERT INTO films SELECT * FROM tmp;\n\nINSERT INTO emp_copy WITH employee AS (select * from\nemp) SELECT * FROM employee;\n\nUPDATE emp_copy SET grp = 'gone' WHERE id =\n(WITH employee AS (select * from emp) SELECT id FROM employee WHERE id\n= 1);\n\nDELETE FROM emp_copy WHERE id IN\n(WITH employee AS (SELECT * FROM emp_copy where grp = 'gone')\nSELECT id FROM employee);\n\n\nWITH manager (mgr_id, mgr_name, mgr_dept) AS\n\t(SELECT id, name, grp\n\tFROM emp_copy\n\tWHERE mgr = id AND grp != 'gone'),\nemployee (emp_id, emp_name, emp_mgr) AS\n\t(SELECT id, name, mgr_id\n\tFROM emp_copy JOIN manager ON grp = mgr_dept),\nmgr_cnt (mgr_id, mgr_reports) AS\n\t(SELECT mgr, COUNT (*)\n\tFROM emp_copy\n\tWHERE mgr != id\n\tGROUP BY mgr)\nSELECT *\nFROM employee JOIN manager ON emp_mgr = mgr_id \n\tJOIN mgr_cnt ON emp_mgr = mgr_id \nWHERE emp_id != mgr_id\nORDER BY mgr_dept;", + "openedge": "-- openedge sample sql\n\nCREATE VIEW ne_customers AS\nSELECT name, address, city, state\nFROM customer\nWHERE state IN ( 'NH', 'MA', 'ME', 'RI', 'CT', 'VT' )\nWITH CHECK OPTION ;\n\nINSERT INTO neworders (order_no, product, qty)\nSELECT order_no, product, qty\nFROM orders\nWHERE order_date = SYSDATE ;\n\nUPDATE OrderLine\nSET (ItemNum, Price) =\n(SELECT ItemNum, Price * 3\nFROM Item\nWHERE ItemName = 'gloves')\nWHERE OrderNum = 21 ;\n\nUpdate Orderline\nSET (Itemnum) =\n(Select Itemnum\nFROM Item\nWHERE Itemname = 'Tennis balls')\nWHERE Ordernum = 20;", + "oracle": "CREATE VIEW vsal \nAS \n SELECT a.deptno \"Department\", \n a.num_emp / b.total_count \"Employees\", \n a.sal_sum / b.total_sal \"Salary\" \n FROM (SELECT deptno, \n Count() num_emp, \n SUM(sal) sal_sum \n FROM scott.emp \n WHERE city = 'NYC' \n GROUP BY deptno) a, \n (SELECT Count() total_count, \n SUM(sal) total_sal \n FROM scott.emp \n WHERE city = 'NYC') b \n;\n\nINSERT ALL\n\tWHEN ottl < 100000 THEN\n\t\tINTO small_orders\n\t\t\tVALUES(oid, ottl, sid, cid)\n\tWHEN ottl > 100000 and ottl < 200000 THEN\n\t\tINTO medium_orders\n\t\t\tVALUES(oid, ottl, sid, cid)\n\tWHEN ottl > 200000 THEN\n\t\tinto large_orders\n\t\t\tVALUES(oid, ottl, sid, cid)\n\tWHEN ottl > 290000 THEN\n\t\tINTO special_orders\nSELECT o.order_id oid, o.customer_id cid, o.order_total ottl,\no.sales_rep_id sid, c.credit_limit cl, c.cust_email cem\nFROM orders o, customers c\nWHERE o.customer_id = c.customer_id;", + "postgresql": "-- postgresql sample sql\n\ncreate view v2 as \nSELECT distributors.name\nFROM distributors\nWHERE distributors.name LIKE 'W%'\nUNION\nSELECT actors.name\nFROM actors\nWHERE actors.name LIKE 'W%';\n\n\nWITH t AS (\nSELECT random() as x FROM generate_series(1, 3)\n)\nSELECT * FROM t\nUNION ALL\nSELECT * FROM t\n;\n\ncreate view v3 as \nWITH RECURSIVE employee_recursive(distance, employee_name, manager_name) AS (\nSELECT 1, employee_name, manager_name\nFROM employee\nWHERE manager_name = 'Mary'\nUNION ALL\nSELECT er.distance + 1, e.employee_name, e.manager_name\nFROM employee_recursive er, employee e\nWHERE er.employee_name = e.manager_name\n)\nSELECT distance, employee_name FROM employee_recursive;\n\nWITH upd AS (\nUPDATE employees SET sales_count = sales_count + 1 WHERE id =\n(SELECT sales_person FROM accounts WHERE name = 'Acme Corporation')\nRETURNING *\n)\nINSERT INTO employees_log SELECT *, current_timestamp FROM upd;\n\n\n/* not implemented\nCREATE RECURSIVE VIEW nums_1_100 (n) AS\nVALUES (1)\nUNION ALL\nSELECT n+1 FROM nums_1_100 WHERE n < 100;\n*/", + "redshift": "-- redshift sample sql\n\ncreate view sales_vw as\nselect * from public.sales\nunion all\nselect * from spectrum.sales\n;\n\ninsert into category_ident(catgroup,catname,catdesc)\nselect catgroup,catname,catdesc from category;\n\nUPDATE category \nSET catdesc = 'Broadway Musical' \nWHERE category.catid IN (SELECT category.catid \n FROM category \n JOIN event \n ON category.catid = event.catid \n JOIN venue \n ON venue.venueid = event.venueid \n JOIN sales \n ON sales.eventid = event.eventid \n WHERE venuecity = 'New York City' \n AND catname = 'Musicals'); \n\nupdate category set catid=100\nfrom event join category cat on event.catid=cat.catid\nwhere cat.catgroup='Concerts';\n\nSELECT qtr, \n Sum(pricepaid) AS qtrsales, \n (SELECT Sum(pricepaid) \n FROM sales \n JOIN date \n ON sales.dateid = date.dateid \n WHERE qtr = '1' \n AND year = 2008) AS q1sales \nFROM sales \n JOIN date \n ON sales.dateid = date.dateid \nWHERE qtr IN( '2', '3' ) \n AND year = 2008 \nGROUP BY qtr \nORDER BY qtr; \n\n\nWITH venue_sales \n AS (SELECT venuename, \n venuecity, \n Sum(pricepaid) AS venuename_sales \n FROM sales, \n venue, \n event \n WHERE venue.venueid = event.venueid \n AND event.eventid = sales.eventid \n GROUP BY venuename, \n venuecity), \n top_venues \n AS (SELECT venuename \n FROM venue_sales \n WHERE venuename_sales > 800000) \nSELECT venuename, \n venuecity, \n venuestate, \n Sum(qtysold) AS venue_qty, \n Sum(pricepaid) AS venue_sales \nFROM sales, \n venue, \n event \nWHERE venue.venueid = event.venueid \n AND event.eventid = sales.eventid \n AND venuename IN(SELECT venuename \n FROM top_venues) \nGROUP BY venuename, \n venuecity, \n venuestate \nORDER BY venuename; ", + "snowflake": "-- snowflake sample sql\n\ncreate or replace secure view myview comment='Test secure view' as select * from mytable;\n\n\ninsert into employees(first_name, last_name, workphone, city,postal_code)\n select\n contractor_first,contractor_last,worknum,null,zip_code\n from contractors\n where contains(worknum,'650');\n \ninsert into employees (first_name,last_name,workphone,city,postal_code)\n with cte as\n (select contractor_first as first_name,contractor_last as last_name,worknum as workphone,city,zip_code as postal_code\n from contractors)\n select first_name,last_name,workphone,city,postal_code\n from cte;\n\ninsert into emp (id,first_name,last_name,city,postal_code,ph)\n select a.id,a.first_name,a.last_name,a.city,a.postal_code,b.ph\n from emp_addr a\n inner join emp_ph b on a.id = b.id;\n\ninsert overwrite all\n into t1\n into t1 (c1, c2, c3) values (n2, n1, default)\n into t2 (c1, c2, c3)\n into t2 values (n3, n2, n1)\nselect n1, n2, n3 from src;\n\ninsert all\n when n1 > 100 then\n into t1\n when n1 > 10 then\n into t1\n into t2\n else\n into t2\nselect n1 from src;\n\ninsert all\ninto t1 values (key, a)\nselect src1.key as key, src1.a as a\nfrom src1, src2 where src1.key = src2.key;\n\n\nmerge into target\n using src on target.k = src.k\n when matched and src.v = 11 then delete\n when matched then update set target.v = src.v;\n\nmerge into target using (select k, max(v) as v from src group by k) as b on target.k = b.k\n when matched then update set target.v = b.v\n when not matched then insert (k, v) values (b.k, b.v);\n\nmerge into members m\n using (\n select id, date\n from signup\n where datediff(day, current_date(), signup.date::date) < -30) s on m.id = s.id\n when matched then update set m.fee = 40;\n\nupdate t1\n set t1.number_column = t1.number_column + t2.number_column, t1.text_column = 'ASDF'\n from t2\n where t1.key_column = t2.t1_key and t1.number_column < 10;\n\nupdate target set v = b.v\n from (select k, min(v) v from src group by k) b\n where target.k = b.k; ", + "mssql": "-- sql server sample sql\nCREATE TABLE dbo.EmployeeSales \n( DataSource varchar(20) NOT NULL, \n BusinessEntityID varchar(11) NOT NULL, \n LastName varchar(40) NOT NULL, \n SalesDollars money NOT NULL \n); \nGO \nCREATE PROCEDURE dbo.uspGetEmployeeSales \nAS \n SET NOCOUNT ON; \n SELECT 'PROCEDURE', sp.BusinessEntityID, c.LastName, \n sp.SalesYTD \n FROM Sales.SalesPerson AS sp \n INNER JOIN Person.Person AS c \n ON sp.BusinessEntityID = c.BusinessEntityID \n WHERE sp.BusinessEntityID LIKE '2%' \n ORDER BY sp.BusinessEntityID, c.LastName; \nGO \n--INSERT...SELECT example \nINSERT INTO dbo.EmployeeSales \n SELECT 'SELECT', sp.BusinessEntityID, c.LastName, sp.SalesYTD \n FROM Sales.SalesPerson AS sp \n INNER JOIN Person.Person AS c \n ON sp.BusinessEntityID = c.BusinessEntityID \n WHERE sp.BusinessEntityID LIKE '2%' \n ORDER BY sp.BusinessEntityID, c.LastName; \nGO \n\n\nCREATE VIEW hiredate_view \nAS \nSELECT p.FirstName, p.LastName, e.BusinessEntityID, e.HireDate \nFROM HumanResources.Employee e \nJOIN Person.Person AS p ON e.BusinessEntityID = p.BusinessEntityID ; \nGO \n\nCREATE VIEW view1 \nAS \nSELECT fis.CustomerKey, fis.ProductKey, fis.OrderDateKey, \n fis.SalesTerritoryKey, dst.SalesTerritoryRegion \nFROM FactInternetSales AS fis \nLEFT OUTER JOIN DimSalesTerritory AS dst \nON (fis.SalesTerritoryKey=dst.SalesTerritoryKey); \n\nGO \nSELECT ROW_NUMBER() OVER(PARTITION BY PostalCode ORDER BY SalesYTD DESC) AS \"Row Number\", \n p.LastName, s.SalesYTD, a.PostalCode \nFROM Sales.SalesPerson AS s \n INNER JOIN Person.Person AS p \n ON s.BusinessEntityID = p.BusinessEntityID \n INNER JOIN Person.Address AS a \n ON a.AddressID = p.BusinessEntityID \nWHERE TerritoryID IS NOT NULL \n AND SalesYTD <> 0 \nORDER BY PostalCode;", + "sybase": "-- sybase sample sql\n\ncreate view accounts (title, advance, amt_due)\nas select title, advance, price * total_sales\nfrom titles\nwhere price > $5\n\ncreate view cities\n(authorname, acity, publishername, pcity)\nas select au_lname, authors.city, pub_name,\npublishers.city\nfrom authors, publishers\nwhere authors.city = publishers.city\n;\n\ncreate view cities2\nas select authorname = au_lname,\nacity = authors.city, publishername = pub_name, pcity =\npublishers.city\nfrom authors, publishers\nwhere authors.city = publishers.city\n;\n\ncreate view psych_titles as\nselect *\nfrom (select * from titles\nwhere type = \"psychology\") dt_psych\n;\n\ninsert newpublishers (pub_id, pub_name)\nselect pub_id, pub_name\nfrom publishers\nwhere pub_name=\"New Age Data\"\n;\n\nmerge into GlobalSales\n(Item_number, Description, Quantity)as G\nusing DailySales as D\nON D.Item_number = G.Item_number\nwhen not matched\nthen\ninsert (Item_number, Description, Quantity )\nvalues (D.Item_number, D.Description, D.Quantity)\nwhen matched\nthen update set\nG.Quantity = G.Quantity + D.Quantity\n;\n\nupdate titles\nset total_sales = total_sales + qty\nfrom titles, salesdetail, sales\nwhere titles.title_id = salesdetail.title_id\nand salesdetail.stor_id = sales.stor_id\nand salesdetail.ord_num = sales.ord_num\nand sales.date in\n(select max (sales.date) from sales)\n;", + "teradata": "-- teradata sample sql\n\nINSERT INTO promotion (deptnum, empname, yearsexp)\nSELECT deptno, name, yrsexp\nFROM employee\nWHERE yrsexp > 10 ;\n\n-- Teradata Database interprets t1 in the scalar subquery as a distinct instance of\n-- t1 rather than as the target table t1 of the insert operation\n/* not supported yet\nINSERT INTO t1 VALUES (1,2,3 (SELECT d2\nFROM t2\nWHERE a2=t1.a1));\n*/\n\n\nINSERT INTO t1\nSELECT a, PERIOD(b, c)\nFROM t2;\n\nUSING (empno INTEGER,\nname VARCHAR(50),\nsalary INTEGER)\nMERGE INTO employee AS t\nUSING VALUES (:empno, :name, :salary) AS s(empno, name, salary)\nON t.empno=s.empno\nWHEN MATCHED THEN UPDATE\nSET salary=s.salary\nWHEN NOT MATCHED THEN INSERT (empno, name, salary)\nVALUES (s.empno, s.name, s.salary);\n\nUSING (empno INTEGER,\nname VARCHAR(50),\nsalary INTEGER)\nUPDATE employee\nSET salary=:salary\nWHERE empno=:empno\nELSE INSERT INTO employee\n(empno, name, salary) VALUES ( :empno, :name, :salary);\n\nUSING (empno INTEGER,\nsalary INTEGER)\nMERGE INTO employee AS t\nUSING (SELECT :empno, :salary, name\nFROM names\nWHERE empno=:empno) AS s(empno, salary, name)\nON t.empno=s.empno\nWHEN MATCHED THEN UPDATE\nSET salary=s.salary, name = s.name\nWHEN NOT MATCHED THEN INSERT (empno, name, salary)\nVALUES (s.empno, s.name, s.salary);\n\nMERGE INTO emp\nUSING VALUES (100, 'cc', 200, 3333) AS emp1 (empnum, name,\ndeptno, sal)\nON emp1.empnum=emp.s_no\nWHEN MATCHED THEN\nUPDATE SET sal=DEFAULT\nWHEN NOT MATCHED THEN\nINSERT VALUES (emp1.empnum, emp1.name, emp1.deptno, emp1.sal);\nMERGE INTO emp\nUSING VALUES (100, 'cc', 200, 3333) AS emp1 (empnum, name,\ndeptno, sal)\nON emp1.empnum=emp.s_no\nWHEN MATCHED THEN\nUPDATE SET sal=DEFAULT(emp.sal)\nWHEN NOT MATCHED THEN\nINSERT VALUES (emp1.empnum, emp1.name, emp1.deptno, emp1.sal)\n;\n\nMERGE INTO t1\nUSING t2\nON x1=z2 AND y1=y2\nWHEN MATCHED THEN\nUPDATE SET z1=10\nWHEN NOT MATCHED THEN\nINSERT (z2,y2,x2);\n\n\nMERGE INTO t1\nUSING (SELECT *\nFROM t2\nWHERE y2=10) AS s\nON x1=10\nWHEN MATCHED THEN\nUPDATE SET z1=z2\nWHEN NOT MATCHED THEN\nINSERT (x2, y2, z2);\n\nUPDATE e\nFROM employee AS e, department AS d\nSET salary = salary * 1.05\nWHERE e.emp_no = d.emp_no;\n\nUPDATE sales_sum_table AS ss\nSET total_sales = (SELECT sum(amount)\nFROM sales_table AS s\nWHERE s.day_of_sale BETWEEN ss.period_start\nAND ss.period_end);\n\n-- The following UPDATE request updates the NoPI table nopi012_t1 aliased as t1.\nUPDATE t1\nFROM nopi012_t1 AS t1, nopi012_t2 AS t2\nSET c3 = t1.c3 * 1.05\nWHERE t1.c2 = t2.c2;", + "vertica": "-- hp vertica sample sql\n\nCREATE VIEW myview AS\nSELECT SUM(annual_income), customer_state\nFROM public.customer_dimension\nWHERE customer_key IN\n(SELECT customer_key\nFROM store.store_sales_fact)\nGROUP BY customer_state\nORDER BY customer_state ASC;\n\nINSERT INTO t1 (col1, col2) (SELECT 'abc', mycolumn FROM mytable);\n\nMERGE INTO t USING s ON (t.c1 = s.c1)\nWHEN NOT MATCHED THEN INSERT (c1, c2) VALUES (s.c1, s.c2);\n\n-- First WITH clause,regional_sales\nWITH\nregional_sales AS (\nSELECT region, SUM(amount) AS total_sales\nFROM orders\nGROUP BY region),\n-- Second WITH clause top_regions\ntop_regions AS (\nSELECT region\nFROM regional_sales\nWHERE total_sales > (SELECT SUM (total_sales)/10 FROM regional_sales) )\n-- End defining WITH clause statement\n-- Begin main primary query\nSELECT region,\nproduct,\nSUM(quantity) AS product_units,\nSUM(amount) AS product_sales\nFROM orders\nWHERE region IN (SELECT region FROM top_regions)\n;" + } +} diff --git a/sqlflowjs/font/Roboto-Regular.ttf b/sqlflowjs/font/Roboto-Regular.ttf new file mode 100644 index 0000000..7d9a6c4 Binary files /dev/null and b/sqlflowjs/font/Roboto-Regular.ttf differ diff --git a/sqlflowjs/font/segoeui-light.woff2 b/sqlflowjs/font/segoeui-light.woff2 new file mode 100644 index 0000000..f2bf6af Binary files /dev/null and b/sqlflowjs/font/segoeui-light.woff2 differ diff --git a/sqlflowjs/font/segoeui-semilight.woff2 b/sqlflowjs/font/segoeui-semilight.woff2 new file mode 100644 index 0000000..c4be6d5 Binary files /dev/null and b/sqlflowjs/font/segoeui-semilight.woff2 differ diff --git a/sqlflowjs/index.html b/sqlflowjs/index.html new file mode 100644 index 0000000..6d7c56a --- /dev/null +++ b/sqlflowjs/index.html @@ -0,0 +1,221 @@ + + + + + + SQLFlow: Visualize column impact and data lineage to track columns across transformations by analyzing SQL query. + + + + + + + + + +
+ + +
+ + + + +
+
+ + + + + + +
+
+

+

+

+

+

+
+
+
+ + + + diff --git a/sqlflowjs/readme.md b/sqlflowjs/readme.md new file mode 100644 index 0000000..29713a6 --- /dev/null +++ b/sqlflowjs/readme.md @@ -0,0 +1,177 @@ +Embed the SQLFlow UI into your application + +# Get started + +Firstly, [download](https://github.com/sqlparser/sqlflow_public/tree/master/sqlflowjs) the sqlflow.js, css and font. + +Secondly, create a new file, insert sqlflow.js and sqlflow.css in the head. + +```html + + + + + + + + + + + + + + + +``` + +Lastly, create a root element, and create a SQLFlow instance on that element. + +```html + + + + + + + + + + + + + +
+ + + + + +``` + +you can [open this simple demo](http://111.229.12.71/sqlflowjs/sqlflow.js_get_start.html) in browser. + + +## Connect to your own SQLFlow backend server +Please modify `config.private.json`, so this sqlflowjs will connect to your own SQLFlow backend server. + +Let's say the IP of your SQLFlow backend server is `192.168.100.50`, set `ApiPrefix` like this: + +``` + "ApiPrefix": "http://192.168.100.50:8083", +``` + +Please also change the token. You can find this token in the SQLFlow backend server, usually it is located under `/wings/sqlflow/backend/user/yourname.json` + +``` + "Authorization": { + "base64token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOiJndWR1c29mdCIsImV4cCI6MTYwMzc1NjgwMCwiaWF0IjoxNTcyMjIwODAwfQ.EhlnJO7oqAHdr0_bunhtrN-TgaGbARKvTh2URTxu9iU" + }, +``` + +## SQLFlow.js api + +you can [open the complete demo](http://111.229.12.71/sqlflowjs/) in browser. + +### SQLFlow(options) + +create a new instance + +options: + +```json +{ + el: HTMLElement, +} +``` + +### SQLFlow.setDbvendor( dbvendor : string ) + +set dbvendor of sql + +dbvendor : "bigquery" | "couchbase" | "db2" | "greenplum" | "hana" | "hive" | "impala" | "informix" | "mysql" | "netezza" | "openedge" | "oracle" | "postgresql" | "redshift" | "snowflake" | "mssql" | "sybase" | "teradata" | "vertica" + +default : "oracle" + +### SQLFlow.setSQLText( sqltext : string ) + +set sql + +### SQLFlow.visualize() + +visualize relations + +### SQLFlow.visualizeJoin() + +visualize join relations + +### SQLFlow.setting + +get current setting of sqlflow instance + +### SQLFlow.setHideAllColumns( value : boolean) + +change sqlflow instance setting + +### SQLFlow.setDataflow( value : boolean) + +change sqlflow instance setting + +### SQLFlow.setImpact( value : boolean) + +change sqlflow instance setting + +### SQLFlow.setShowIntermediateRecordset( value : boolean) + +change sqlflow instance setting + +### SQLFlow.setShowFunction( value : boolean) + +change sqlflow instance setting diff --git a/sqlflowjs/sqlflow.css b/sqlflowjs/sqlflow.css new file mode 100644 index 0000000..eb906c7 --- /dev/null +++ b/sqlflowjs/sqlflow.css @@ -0,0 +1,932 @@ +.font-Roboto{ + font-family:'Roboto', Arial, sans-serif; +} +.no-select{ + -webkit-touch-callout:none; + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +.disable-user-select{ + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +@font-face{ + font-family:'Roboto'; + src:url('font/Roboto-Regular.ttf'); +} +@font-face{ + font-family:'Segoe UI'; + src:url('font/segoeui-light.woff2'); +} +@font-face{ + font-family:'Segoe UI Bold'; + src:url('font/segoeui-semilight.woff2'); +} +div.SQLFlowGraph{ + font-family:Arial, sans-serif; + -webkit-font-smoothing:antialiased; +} +div.SQLFlowGraph *{ + margin:0; + padding:0; + box-sizing:border-box; +} +div.SQLFlowGraph ul, +div.SQLFlowGraph ol{ + list-style:none; +} +div.SQLFlowGraph hr{ + border:0; +} +div.SQLFlowGraph h1, +div.SQLFlowGraph h2, +div.SQLFlowGraph h3, +div.SQLFlowGraph h4, +div.SQLFlowGraph h5, +div.SQLFlowGraph h6{ + font-weight:normal; + cursor:default; +} +div.SQLFlowGraph h1{ + font-size:12px; +} +div.SQLFlowGraph h2{ + font-size:14px; +} +div.SQLFlowGraph i{ + font-style:normal; +} +div.SQLFlowGraph address{ + font-style:normal; +} +div.SQLFlowGraph button{ + cursor:pointer; +} +div.SQLFlowGraph a{ + cursor:pointer; +} +div.SQLFlowGraph a:hover, +div.SQLFlowGraph a:link, +div.SQLFlowGraph a:visited, +div.SQLFlowGraph a:active{ + text-decoration:none; + color:black; +} +.hide{ + display:none !important; +} + +.Flex{ + display:-ms-flexbox; + display:flex; +} +.Flex.hide{ + display:none; +} +.Flex.Col{ + -ms-flex-direction:column; + flex-direction:column; +} +.Flex.ColCenter{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:center; + align-items:center; +} +.Flex.ColCenterCenter{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:center; + align-items:center; + -ms-flex-pack:center; + justify-content:center; +} +.Flex.ColCenterEnd{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:end; + align-items:flex-end; + -ms-flex-pack:center; + justify-content:center; +} +.Flex.ColStartBetween{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:start; + align-items:flex-start; + -ms-flex-pack:justify; + justify-content:space-between; +} +.Flex.ColStartCenter{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:start; + align-items:flex-start; + -ms-flex-pack:center; + justify-content:center; +} +.Flex.ColCenterBetween{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:center; + align-items:center; + -ms-flex-pack:justify; + justify-content:space-between; +} +.Flex.ColStretchCenter{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-pack:center; + justify-content:center; +} +.Flex.ColStretchBetween{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-pack:justify; + justify-content:space-between; +} +.Flex.ColBetweenCenter{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:space-between; + align-items:space-between; + -ms-flex-pack:center; + justify-content:center; +} +.Flex.ColEnd{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:end; + align-items:flex-end; +} +.Flex.ColEnd{ + -ms-flex-direction:column; + flex-direction:column; + -ms-flex-align:end; + align-items:flex-end; +} +.Flex.RowBetween{ + -ms-flex-pack:justify; + justify-content:space-between; +} +.Flex.RowBetweenCenter{ + -ms-flex-pack:justify; + justify-content:space-between; + -ms-flex-align:center; + align-items:center; +} +.Flex.RowAround{ + -ms-flex-pack:distribute; + justify-content:space-around; +} +.Flex.RowAroundCenter{ + -ms-flex-pack:distribute; + justify-content:space-around; + -ms-flex-align:center; + align-items:center; +} +.Flex.RowEvenly{ + -ms-flex-pack:space-evenly; + justify-content:space-evenly; +} +.Flex.RowEvenlyCenter{ + -ms-flex-pack:space-evenly; + justify-content:space-evenly; + -ms-flex-align:center; + align-items:center; +} +.Flex.RowStartCenter{ + -ms-flex-align:center; + align-items:center; +} +.Flex.RowStartEnd{ + -ms-flex-align:end; + align-items:flex-end; +} +.Flex.RowCenter{ + -ms-flex-pack:center; + justify-content:center; +} +.Flex.RowCenterCenter{ + -ms-flex-pack:center; + justify-content:center; + -ms-flex-align:center; + align-items:center; +} +.Flex.RowCenterEnd{ + -ms-flex-pack:center; + justify-content:center; + -ms-flex-align:end; + align-items:flex-end; +} +.Flex.RowEnd{ + -ms-flex-pack:end; + justify-content:flex-end; +} +.Flex.RowEndCenter{ + -ms-flex-pack:end; + justify-content:flex-end; + -ms-flex-align:center; + align-items:center; +} + +div.ImageCover{ + background-position:center center; + background-size:cover; + background-repeat:no-repeat; +} +div.ImageContain{ + background-position:center center; + background-size:contain; + background-repeat:no-repeat; +} + +div.Overlay{ + position:fixed; + z-index:50; + top:0; + left:0; + width:100%; + height:100%; + background-color:rgba(0, 0, 0, 0.7); +} +div.Overlay.enter{ + opacity:0; +} +div.Overlay.enter-active{ + opacity:1; + transition:all 0.3s; +} +div.Overlay.exit{ + opacity:1; +} +div.Overlay.exit-active{ + opacity:0; + transition:all 0.3s; +} + +div.Dialog.enter{ + opacity:0; + -webkit-transform:translate3d(-50%, -50%, 0) scale(0.7); + transform:translate3d(-50%, -50%, 0) scale(0.7); +} +div.Dialog.enter-active{ + opacity:1; + -webkit-transform:translate3d(-50%, -50%, 0); + transform:translate3d(-50%, -50%, 0); + transition:all 0.3s; +} +div.Dialog.exit{ + opacity:1; + -webkit-transform:translate3d(-50%, -50%, 0); + transform:translate3d(-50%, -50%, 0); +} +div.Dialog.exit-active{ + opacity:0; + -webkit-transform:translate3d(-50%, -50%, 0) scale(0.9); + transform:translate3d(-50%, -50%, 0) scale(0.9); + transition:all 0.3s; +} + +div.Close{ + color:grey; + cursor:pointer; +} +div.Close:hover{ + color:#000; +} +div.Close > svg{ + width:100%; + height:100%; + fill:currentColor; +} + +.font-Roboto{ + font-family:'Roboto', Arial, sans-serif; +} +.no-select{ + -webkit-touch-callout:none; + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +.disable-user-select{ + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +@-webkit-keyframes MoveOut{ + 0%{ + max-height:150px; + padding:8px; + opacity:1; + } + 100%{ + max-height:0; + padding:0; + opacity:0; + } +} +@keyframes MoveOut{ + 0%{ + max-height:150px; + padding:8px; + opacity:1; + } + 100%{ + max-height:0; + padding:0; + opacity:0; + } +} +ul.Notification{ + position:fixed; + z-index:200; + left:50%; + bottom:25px; + -webkit-transform:translate3d(-50%, 0, 0); + transform:translate3d(-50%, 0, 0); +} +ul.Notification > li{ + position:relative; + min-height:95px; + max-height:205px; + margin-bottom:20px; + padding:16px 24px; + background:#fff; + border-radius:6px; + box-shadow:0 0 21px 3px rgba(0, 0, 0, 0.15); + display:-ms-flexbox; + display:flex; + -webkit-animation-timing-function:cubic-bezier(0.6, 0.04, 0.98, 0.34); + animation-timing-function:cubic-bezier(0.6, 0.04, 0.98, 0.34); +} +ul.Notification > li > div{ + margin-left:24px; + width:270px; +} +ul.Notification > li > div > h1{ + color:rgba(0, 0, 0, 0.85); + font-size:16px; + line-height:24px; + margin-bottom:8px; + font-weight:bold; + -ms-flex:none; + flex:none; +} +ul.Notification > li > div > h2{ + padding-right:10px; + width:100%; + font-size:14px; + line-height:20px; + color:#a1a2a2; + white-space:pre-line; + word-break:break-word; + overflow:auto; + max-height:140px; +} +ul.Notification > li > div > h2::-webkit-scrollbar{ + width:6px; + height:6px; +} +ul.Notification > li > div > h2::-webkit-scrollbar-thumb{ + border-radius:15px; + min-height:15px; + -webkit-transition:all 0.3s; + transition:all 0.3s; + border-radius:6px; + background-color:#ccc; +} +ul.Notification > li > svg{ + width:24px; + height:24px; + -ms-flex:none; + flex:none; +} +ul.Notification > li > svg.Error{ + fill:red; +} +ul.Notification > li > svg.Attention{ + fill:#1890ff; +} +ul.Notification > li > div.Close{ + position:absolute; + top:14px; + right:12px; + width:15px; + height:15px; +} + +div.Arrow > svg{ + width:100%; + height:100%; +} +div.Arrow.ArrowRight{ + -webkit-transform:rotate(90deg); + transform:rotate(90deg); +} +div.Arrow.ArrowDown{ + -webkit-transform:rotate(180deg); + transform:rotate(180deg); +} +div.Arrow.ArrowLeft{ + -webkit-transform:rotate(270deg); + transform:rotate(270deg); +} + +div.Switcher{ + width:40px; + height:18px; + background-color:rgba(0, 0, 0, 0.25); + border-radius:100px; + display:-ms-flexbox; + display:flex; + -ms-flex-align:center; + align-items:center; + position:relative; + cursor:pointer; +} +div.Switcher.checked{ + background-color:#36b999; +} +div.Switcher.checked > div{ + left:auto; + right:3px; +} +div.Switcher.disabled{ + cursor:not-allowed; + opacity:0.4; +} +div.Switcher > div{ + position:absolute; + top:3px; + left:3px; + box-shadow:0 2px 4px 0 rgba(0, 35, 11, 0.2); + width:12px; + height:12px; + background-color:#fff; + border-radius:18px; +} + +.font-Roboto{ + font-family:'Roboto', Arial, sans-serif; +} +.no-select{ + -webkit-touch-callout:none; + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +.disable-user-select{ + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +span.Loading > svg{ + width:100%; + height:100%; + -webkit-animation:rotate 2s linear infinite; + animation:rotate 2s linear infinite; +} +span.Loading > svg > circle{ + stroke:currentColor; + stroke-width:3; + stroke-linecap:round; + -webkit-animation:circular 1.5s ease-in-out infinite; + animation:circular 1.5s ease-in-out infinite; +} +@-webkit-keyframes circular{ + 0%{ + stroke-dasharray:1, 200; + stroke-dashoffset:0; + } + 50%{ + stroke-dasharray:90, 150; + stroke-dashoffset:-40; + } + 100%{ + stroke-dasharray:90, 150; + stroke-dashoffset:-120; + } +} +@keyframes circular{ + 0%{ + stroke-dasharray:1, 200; + stroke-dashoffset:0; + } + 50%{ + stroke-dasharray:90, 150; + stroke-dashoffset:-40; + } + 100%{ + stroke-dasharray:90, 150; + stroke-dashoffset:-120; + } +} +@-webkit-keyframes rotate{ + 0%{ + -webkit-transform:rotate(0deg); + transform:rotate(0deg); + } + 100%{ + -webkit-transform:rotate(360deg); + transform:rotate(360deg); + } +} +@keyframes rotate{ + 0%{ + -webkit-transform:rotate(0deg); + transform:rotate(0deg); + } + 100%{ + -webkit-transform:rotate(360deg); + transform:rotate(360deg); + } +} + +div.Code{ + position:fixed; + z-index:10; + right:0; + bottom:0; + width:100%; + background-color:white; +} +div.Code.enter{ + -webkit-transform:translate3d(0, 100%, 0); + transform:translate3d(0, 100%, 0); +} +div.Code.enter-active{ + -webkit-transform:translate3d(0, 0, 0); + transform:translate3d(0, 0, 0); + transition:all 0.3s; +} +div.Code.exit{ + -webkit-transform:translate3d(0, 0, 0); + transform:translate3d(0, 0, 0); +} +div.Code.exit-active{ + -webkit-transform:translate3d(0, 100%, 0); + transform:translate3d(0, 100%, 0); + transition:all 0.3s; +} +div.Code > div.Header{ + height:40px; + background-color:#dcdcdc; + position:relative; +} +div.Code > div.Header > div.Drag{ + position:absolute; + height:6px; + top:-3px; + left:0; + right:0; + background:transparent; + cursor:n-resize; +} +div.Code > div.Header > div.Title{ + position:absolute; + left:20px; + bottom:0; + background-color:#f8f8f8; + padding:8px 10px; + border-radius:5px 5px 0 0; + font-size:14px; +} +div.Code > div.Header > div.Close{ + position:absolute; + top:10px; + right:7px; + width:20px; + height:20px; +} +div.Code > div.Operation{ + height:20px; + padding-left:30px; + background-color:#f8f8f8; +} +div.Code > div.Operation > p{ + margin-right:15px; + font-size:14px; +} +div.Code > div.Operation > svg{ + height:20px; + width:20px; + margin-right:15px; + fill:#545454; + cursor:pointer; +} +div.Code > div.Operation > svg.disabled{ + cursor:default; + fill:#b5b5b5; +} +div.Code > div.CodeMirror{ + height:calc(100% - 40px - 20px); +} +div.Code > div.CodeMirror div.CodeMirror{ + height:100%; +} +div.Code > div.CodeMirror div.CodeMirror .highlight{ + background:#443b3b; + color:#ffffff; +} +div.Code > div.CodeMirror div.CodeMirror .mouseover{ + background:#dadada; + color:#ffffff; +} + +.font-Roboto{ + font-family:'Roboto', Arial, sans-serif; +} +.no-select{ + -webkit-touch-callout:none; + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +.disable-user-select{ + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +div.SQLFlowGraph{ + height:100%; + overflow:auto; + position:relative; +} +div.SQLFlowGraph > h1{ + height:40px; + background-color:#e7e5e5; + border-bottom:1px solid #d2d2d2; +} +div.SQLFlowGraph > h2{ + height:35px; + background-color:#f7f7f7; + border-bottom:1px solid #e3dfdf; +} +div.SQLFlowGraph > p{ + font-family:"Roboto", sans-serif; + font-size:15px; + color:#929292; + padding:20px 35px; +} +div.SQLFlowGraph > div.Overlay{ + min-width:100%; + min-height:100%; + position:absolute; + top:0; + left:0; + z-index:15; + background-color:rgba(226, 226, 226, 0.78); +} +div.SQLFlowGraph > div.Overlay > div{ + position:absolute; + top:50%; + left:50%; + -webkit-transform:translate3d(-50%, -50%, 0); + transform:translate3d(-50%, -50%, 0); +} +div.SQLFlowGraph > div.Overlay > div > h1{ + text-align:center; + color:#464545; +} +div.SQLFlowGraphZoom{ + position:fixed; + z-index:5; + top:115px; + right:30px; + cursor:default; +} +div.SQLFlowGraphZoom > div{ + height:25px; + width:25px; + padding:5px; + border-radius:5px; + background-color:rgba(238, 238, 238, 0.65); + margin-bottom:25px; +} +div.SQLFlowGraphZoom > div > svg{ + height:100%; + width:100%; + fill:#aaaaaa; + cursor:pointer; +} +div.SQLFlowGraphZoom > div > svg:hover{ + fill:#000; +} +div#SQLFlowGraphTables{ + position:relative; +} +div#SQLFlowGraphTables > svg{ + width:100%; + height:100%; +} +div#SQLFlowGraphTables > svg > g > path{ + stroke:#ababab; + stroke-width:1; + fill:none; +} +div#SQLFlowGraphTables > svg > g > path.join{ + stroke:#a9ce76; +} +div#SQLFlowGraphTables > svg > g > polygon{ + stroke:#ababab; + fill:#ababab; +} +div#SQLFlowGraphTables > svg > g > text{ + font-size:14px; +} +div#SQLFlowGraphTables > svg > g.highlight > path{ + stroke-width:1.4; + stroke:#000000; +} +div#SQLFlowGraphTables > svg > g.highlight > path.join{ + stroke:#65ab04; +} +div#SQLFlowGraphTables > svg > g.highlight > polygon{ + stroke:#000000; + fill:#000000; +} +div#SQLFlowGraphTables > svg > g.dash > path{ + stroke-dasharray:4 2; +} +div.SQLFlowGraphTable{ + position:absolute; + border-radius:3px; + -webkit-user-select:none; + -moz-user-select:none; + -ms-user-select:none; + user-select:none; +} +div.SQLFlowGraphTable.active{ + z-index:5; +} +div.SQLFlowGraphTable.table{ + border:1px solid #91c051; + background-color:#91c051; +} +div.SQLFlowGraphTable.view{ + border:1px solid #419A5F; + background-color:#419A5F; +} +div.SQLFlowGraphTable.select_list{ + border:1px solid #D26B58; + background-color:#D26B58; +} +div.SQLFlowGraphTable.insert-select{ + border:1px solid #B24B78; + background-color:#B24B78; +} +div.SQLFlowGraphTable.update-set{ + border:1px solid #D29059; + background-color:#D29059; +} +div.SQLFlowGraphTable.merge-insert{ + border:1px solid #893A89; + background-color:#893A89; +} +div.SQLFlowGraphTable.merge-update{ + border:1px solid #d28f59; + background-color:#d28f59; +} +div.SQLFlowGraphTable.select_union{ + border:1px solid #357E7E; + background-color:#357E7E; +} +div.SQLFlowGraphTable.with_cte{ + border:1px solid #FF6E97; + background-color:#FF6E97; +} +div.SQLFlowGraphTable.update-select{ + border:1px solid #5583d8; + background-color:#5583d8; +} +div.SQLFlowGraphTable.function{ + border:1px solid #a0a0a0; + background-color:#a0a0a0; +} +div.SQLFlowGraphTable > h1{ + height:22px; +} +div.SQLFlowGraphTable > h1 > span{ + text-align:center; + text-overflow:ellipsis; + overflow:hidden; + display:block; + -webkit-transform:scale(0.916); + transform:scale(0.916); + font-family:verdana, sans serif; + font-size:12px; + line-height:22px; + color:white; + height:100%; + word-break:break-all; +} +div.SQLFlowGraphTable > ul{ + background-color:#ffffff; +} +div.SQLFlowGraphTable > ul > li{ + height:16px; + cursor:default; +} +div.SQLFlowGraphTable > ul > li > span{ + text-overflow:ellipsis; + overflow:hidden; + display:block; + font-family:verdana, sans serif; + font-size:12px; + -webkit-transform:scale(0.916); + transform:scale(0.916); + height:100%; + word-break:break-all; +} +div.SQLFlowGraphTable > ul > li.highlight{ + background-color:#dddddd; +} +div.SQLFlowGraphTable > ul > li.mouseover{ + background-color:#faebd7; +} +div.SQLFlowGraphMap{ + position:fixed; + background-color:#fff; + border:1px solid #000; + width:104px; + height:104px; + opacity:0.9; + overflow:hidden; + z-index:10; + box-sizing:content-box; + cursor:default; +} +div.SQLFlowGraphMap div.Viewport{ + z-index:1; + left:0px; + top:0px; + position:absolute; + border:2px solid #f00; + cursor:move; + box-sizing:content-box; +} +div.SQLFlowGraphMapTable{ + position:absolute; + border:1px solid #000; + background-color:#eea; +} +div.SQLFlowGraphMapTable.active{ + background-color:#f85; +} +div.SQLFlowGraphContextMenu{ + position:fixed; + min-width:160px; + min-height:110px; + border:1px solid #666666; + background:white; + z-index:15; + box-shadow:2px 2px 5px #888888; + border-radius:3px; + cursor:default; +} +div.SQLFlowGraphContextMenu > h1{ + height:22px; + line-height:22px; + padding:0 8px; + background:#666666; + text-align:center; + color:white; + font-size:14px; +} +div.SQLFlowGraphContextMenu > ul > li{ + color:#000000; + padding:2px 0; +} +div.SQLFlowGraphContextMenu > ul > li.Download{ + border-top:1px solid #e8dada; +} +div.SQLFlowGraphContextMenu > ul > li.disable > h1{ + cursor:default; + color:grey; +} +div.SQLFlowGraphContextMenu > ul > li.disable > h1:hover{ + background-color:white; +} +div.SQLFlowGraphContextMenu > ul > li > h1{ + padding:0 8px; + height:22px; + line-height:22px; + font-size:14px; + cursor:pointer; +} +div.SQLFlowGraphContextMenu > ul > li > h1:hover{ + background-color:#dddddd; +} + diff --git a/sqlflowjs/sqlflow.js b/sqlflowjs/sqlflow.js new file mode 100644 index 0000000..89aad23 --- /dev/null +++ b/sqlflowjs/sqlflow.js @@ -0,0 +1 @@ +var SQLFlow=function(n){var r={};function i(e){if(r[e])return r[e].exports;var t=r[e]={i:e,l:!1,exports:{}};return n[e].call(t.exports,t,t.exports,i),t.l=!0,t.exports}return i.m=n,i.c=r,i.d=function(e,t,n){i.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(t,e){if(1&e&&(t=i(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var n=Object.create(null);if(i.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var r in t)i.d(n,r,function(e){return t[e]}.bind(null,r));return n},i.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(t,"a",t),t},i.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},i.p="",i(i.s=493)}([function(e,t,n){"use strict";e.exports=n(410)},function(e,t,n){var v=n(6),m=n(20),g=n(24),y=n(25),b=n(29),w="prototype",E=function(e,t,n){var r,i,o,a,s=e&E.F,l=e&E.G,u=e&E.S,c=e&E.P,f=e&E.B,d=l?v:u?v[t]||(v[t]={}):(v[t]||{})[w],p=l?m:m[t]||(m[t]={}),h=p[w]||(p[w]={});for(r in l&&(n=t),n)o=((i=!s&&d&&void 0!==d[r])?d:n)[r],a=f&&i?b(o,v):c&&"function"==typeof o?b(Function.call,o):o,d&&y(d,r,o,e&E.U),p[r]!=o&&g(p,r,a),c&&h[r]!=o&&(h[r]=o)};v.core=m,E.F=1,E.G=2,E.S=4,E.P=8,E.B=16,E.W=32,E.U=64,E.R=128,e.exports=E},function(e,t){e.exports=function(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}},function(e,t,n){e.exports=n(198)},function(e,t,n){var r=n(7);e.exports=function(e){if(!r(e))throw TypeError(e+" is not an object!");return e}},function(e,t){function l(e,t,n,r,i,o,a){try{var s=e[o](a),l=s.value}catch(e){return void n(e)}s.done?t(l):Promise.resolve(l).then(r,i)}e.exports=function(s){return function(){var e=this,a=arguments;return new Promise(function(t,n){var r=s.apply(e,a);function i(e){l(r,t,n,i,o,"next",e)}function o(e){l(r,t,n,i,o,"throw",e)}i(void 0)})}}},function(e,t){var n=e.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=n)},function(e,t){e.exports=function(e){return"object"==typeof e?null!==e:"function"==typeof e}},function(e,t){e.exports=function(e){try{return!!e()}catch(e){return!0}}},function(e,t,n){function i(e){if(o[e])return o[e].exports;var t=o[e]={i:e,l:!1,exports:{}};return r[e].call(t.exports,t,t.exports,i),t.l=!0,t.exports}var r,o;e.exports=(o={},i.m=r=[function(e,t){e.exports=function(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}},function(e,t){e.exports=n(132)},function(e,t){function r(e,t){for(var n=0;n"+i+""}var i=n(1),o=n(8),a=n(39),s=/"/g;e.exports=function(t,e){var n={};n[t]=e(r),i(i.P+i.F*o(function(){var e=""[t]('"');return e!==e.toLowerCase()||3document.F=Object<\/script>"),e.close(),c=e.F;n--;)delete c[u][s[n]];return c()};e.exports=Object.create||function(e,t){var n;return null!==e?(i[u]=o(e),n=new i,i[u]=null,n[l]=e):n=c(),void 0===t?n:a(n,t)}},function(e,t,n){"use strict";if(n(12)){var g=n(48),y=n(6),b=n(8),w=n(1),E=n(88),r=n(122),h=n(29),x=n(59),i=n(47),S=n(24),o=n(60),a=n(33),T=n(13),O=n(162),s=n(55),l=n(38),u=n(23),C=n(50),k=n(7),v=n(17),m=n(111),_=n(44),A=n(27),I=n(56).f,N=n(70),c=n(54),f=n(10),d=n(41),p=n(76),L=n(72),R=n(115),P=n(57),M=n(83),D=n(58),j=n(114),F=n(152),H=n(15),W=n(26),U=H.f,z=W.f,B=y.RangeError,V=y.TypeError,G=y.Uint8Array,q="ArrayBuffer",Y="Shared"+q,X="BYTES_PER_ELEMENT",K="prototype",$=Array[K],Q=r.ArrayBuffer,J=r.DataView,Z=d(0),ee=d(2),te=d(3),ne=d(4),re=d(5),ie=d(6),oe=p(!0),ae=p(!1),se=R.values,le=R.keys,ue=R.entries,ce=$.lastIndexOf,fe=$.reduce,de=$.reduceRight,pe=$.join,he=$.sort,ve=$.slice,me=$.toString,ge=$.toLocaleString,ye=f("iterator"),be=f("toStringTag"),we=c("typed_constructor"),Ee=c("def_constructor"),xe=E.CONSTR,Se=E.TYPED,Te=E.VIEW,Oe="Wrong length!",Ce=d(1,function(e,t){return Ne(L(e,e[Ee]),t)}),ke=b(function(){return 1===new G(new Uint16Array([1]).buffer)[0]}),_e=!!G&&!!G[K].set&&b(function(){new G(1).set({})}),Ae=function(e,t){var n=a(e);if(n<0||n%t)throw B("Wrong offset!");return n},Ie=function(e){if(k(e)&&Se in e)return e;throw V(e+" is not a typed array!")},Ne=function(e,t){if(!(k(e)&&we in e))throw V("It is not a typed array constructor!");return new e(t)},Le=function(e,t){return Re(L(e,e[Ee]),t)},Re=function(e,t){for(var n=0,r=t.length,i=Ne(e,r);n")}),y=function(){var e=/(?:)/,t=e.exec;e.exec=function(){return t.apply(this,arguments)};var n="ab".split(e);return 2===n.length&&"a"===n[0]&&"b"===n[1]}();e.exports=function(n,e,t){var r=h(n),o=!d(function(){var e={};return e[r]=function(){return 7},7!=""[n](e)}),i=o?!d(function(){var e=!1,t=/a/;return t.exec=function(){return e=!0,null},"split"===n&&(t.constructor={},t.constructor[m]=function(){return t}),t[r](""),!e}):void 0;if(!o||!i||"replace"===n&&!g||"split"===n&&!y){var a=/./[r],s=t(p,r,""[n],function(e,t,n,r,i){return t.exec===v?o&&!i?{done:!0,value:a.call(t,n,r)}:{done:!0,value:e.call(n,t,r)}:{done:!1}}),l=s[0],u=s[1];c(String.prototype,n,l),f(RegExp.prototype,r,2==e?function(e,t){return u.call(e,this,t)}:function(e){return u.call(e,this)})}}},function(e,t,n){var r=n(6).navigator;e.exports=r&&r.userAgent||""},function(e,t,n){"use strict";var g=n(6),y=n(1),b=n(25),w=n(60),E=n(49),x=n(52),S=n(59),T=n(7),O=n(8),C=n(83),k=n(63),_=n(105);e.exports=function(r,e,t,n,i,o){function a(e){var n=c[e];b(c,e,"delete"==e?function(e){return!(o&&!T(e))&&n.call(this,0===e?0:e)}:"has"==e?function(e){return!(o&&!T(e))&&n.call(this,0===e?0:e)}:"get"==e?function(e){return o&&!T(e)?void 0:n.call(this,0===e?0:e)}:"add"==e?function(e){return n.call(this,0===e?0:e),this}:function(e,t){return n.call(this,0===e?0:e,t),this})}var s=g[r],l=s,u=i?"set":"add",c=l&&l.prototype,f={};if("function"==typeof l&&(o||c.forEach&&!O(function(){(new l).entries().next()}))){var d=new l,p=d[u](o?{}:-0,1)!=d,h=O(function(){d.has(1)}),v=C(function(e){new l(e)}),m=!o&&O(function(){for(var e=new l,t=5;t--;)e[u](t,t);return!e.has(-0)});v||(((l=e(function(e,t){S(e,l,r);var n=_(new s,e,l);return null!=t&&x(t,i,n[u],n),n})).prototype=c).constructor=l),(h||m)&&(a("delete"),a("has"),i&&a("get")),(m||p)&&a(u),o&&c.clear&&delete c.clear}else l=n.getConstructor(e,r,i,u),w(l.prototype,t),E.NEED=!0;return k(l,r),f[r]=l,y(y.G+y.W+y.F*(l!=s),f),o||n.setStrong(l,r,i),l}},function(e,t,n){for(var r,i=n(6),o=n(24),a=n(54),s=a("typed_array"),l=a("view"),u=!(!i.ArrayBuffer||!i.DataView),c=u,f=0,d="Int8Array,Uint8Array,Uint8ClampedArray,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array".split(",");f<9;)(r=i[d[f++]])?(o(r.prototype,s,!0),o(r.prototype,l,!0)):c=!1;e.exports={ABV:u,CONSTR:c,TYPED:s,VIEW:l}},function(e,t,n){"use strict";e.exports=n(48)||!n(8)(function(){var e=Math.random();__defineSetter__.call(null,e,function(){}),delete n(6)[e]})},function(e,t,n){"use strict";var r=n(1);e.exports=function(e){r(r.S,e,{of:function(){for(var e=arguments.length,t=new Array(e);e--;)t[e]=arguments[e];return new this(t)}})}},function(e,t,n){"use strict";var r=n(1),l=n(19),u=n(29),c=n(52);e.exports=function(e){r(r.S,e,{from:function(e,t,n){var r,i,o,a,s=t;return l(this),(r=void 0!==s)&&l(s),null==e?new this:(i=[],r?(o=0,a=u(s,n,2),c(e,!1,function(e){i.push(a(e,o++))})):c(e,!1,i.push,i),new this(i))}})}},function(e,t,n){e.exports=n(425)},function(e,Ee,xe){"use strict";xe.r(Ee),function(e){var t;xe.d(Ee,"Immer",function(){return ue}),xe.d(Ee,"applyPatches",function(){return me}),xe.d(Ee,"castDraft",function(){return be}),xe.d(Ee,"castImmutable",function(){return we}),xe.d(Ee,"createDraft",function(){return ge}),xe.d(Ee,"finishDraft",function(){return ye}),xe.d(Ee,"immerable",function(){return s}),xe.d(Ee,"isDraft",function(){return v}),xe.d(Ee,"isDraftable",function(){return m}),xe.d(Ee,"nothing",function(){return f}),xe.d(Ee,"original",function(){return g}),xe.d(Ee,"produce",function(){return de}),xe.d(Ee,"produceWithPatches",function(){return pe}),xe.d(Ee,"setAutoFreeze",function(){return he}),xe.d(Ee,"setUseProxies",function(){return ve});var u,n,c,r,i="undefined"!=typeof Symbol,o="undefined"!=typeof Map,a="undefined"!=typeof Set,f=i?Symbol("immer-nothing"):((t={})["immer-nothing"]=!0,t),s=i?Symbol("immer-draftable"):"__$immer_draftable",d=i?Symbol("immer-state"):"__$immer_state",l=i?Symbol.iterator:"@@iterator",p=function(e,t){return(p=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])})(e,t)};function h(e,t){function n(){this.constructor=e}p(e,t),e.prototype=(n.prototype=t.prototype,new n)}function v(e){return!!e&&!!e[d]}function m(e){return!!e&&(function(e){if(!e||"object"!=typeof e)return!1;var t=Object.getPrototypeOf(e);return!t||t===Object.prototype}(e)||Array.isArray(e)||!!e[s]||!!e.constructor[s]||O(e)||C(e))}function g(e){if(e&&e[d])return e[d].base}(n=u=u||{})[n.Object=0]="Object",n[n.Array=1]="Array",n[n.Map=2]="Map",n[n.Set=3]="Set",(r=c=c||{})[r.ProxyObject=0]="ProxyObject",r[r.ProxyArray=1]="ProxyArray",r[r.ES5Object=2]="ES5Object",r[r.ES5Array=3]="ES5Array",r[r.Map=4]="Map",r[r.Set=5]="Set";var y="undefined"!=typeof Reflect&&Reflect.ownKeys?Reflect.ownKeys:void 0!==Object.getOwnPropertySymbols?function(e){return Object.getOwnPropertyNames(e).concat(Object.getOwnPropertySymbols(e))}:Object.getOwnPropertyNames;function b(n,r){w(n)===u.Object?y(n).forEach(function(e){return r(e,n[e],n)}):n.forEach(function(e,t){return r(t,e,n)})}function w(e){if(e||N(),e[d])switch(e[d].type){case c.ES5Object:case c.ProxyObject:return u.Object;case c.ES5Array:case c.ProxyArray:return u.Array;case c.Map:return u.Map;case c.Set:return u.Set}return Array.isArray(e)?u.Array:O(e)?u.Map:C(e)?u.Set:u.Object}function E(e,t){return w(e)===u.Map?e.has(t):Object.prototype.hasOwnProperty.call(e,t)}function x(e,t){return w(e)===u.Map?e.get(t):e[t]}function S(e,t,n){switch(w(e)){case u.Map:e.set(t,n);break;case u.Set:e.delete(t),e.add(n);break;default:e[t]=n}}function T(e,t){return e===t?0!==e||1/e==1/t:e!=e&&t!=t}function O(e){return o&&e instanceof Map}function C(e){return a&&e instanceof Set}function k(e){return e.copy||e.base}function _(r,i){if(void 0===i&&(i=!1),Array.isArray(r))return r.slice();var o=Object.create(Object.getPrototypeOf(r));return y(r).forEach(function(e){if(e!==d){var t=Object.getOwnPropertyDescriptor(r,e),n=t.value;if(t.get){if(!i)throw new Error("Immer drafts cannot have computed properties");n=t.get.call(r)}t.enumerable?o[e]=n:Object.defineProperty(o,e,{value:n,writable:!0,configurable:!0})}}),o}function A(e,t){if(m(e)&&!v(e)&&!Object.isFrozen(e)){var n=w(e);n===u.Set?e.add=e.clear=e.delete=I:n===u.Map&&(e.set=e.clear=e.delete=I),Object.freeze(e),t&&b(e,function(e,t){return A(t,!0)})}}function I(){throw new Error("This object has been frozen and should not be mutated")}function N(){throw new Error("Illegal state, please file a bug")}var L=(R.prototype.usePatches=function(e){e&&(this.patches=[],this.inversePatches=[],this.patchListener=e)},R.prototype.revoke=function(){this.leave(),this.drafts.forEach(P),this.drafts=null},R.prototype.leave=function(){this===R.current&&(R.current=this.parent)},R.enter=function(e){var t=new R(R.current,e);return R.current=t},R);function R(e,t){this.drafts=[],this.parent=e,this.immer=t,this.canAutoFreeze=!0}function P(e){var t=e[d];t.type===c.ProxyObject||t.type===c.ProxyArray?t.revoke():t.revoked=!0}function M(e,t,n){var r=n.drafts[0],i=void 0!==t&&t!==r;if(e.willFinalize(n,t,i),i){if(r[d].modified)throw n.revoke(),new Error("An immer producer returned a new value *and* modified its draft. Either return a new value *or* modify the draft.");m(t)&&(t=D(e,t,n),n.parent||F(e,t)),n.patches&&(n.patches.push({op:"replace",path:[],value:t}),n.inversePatches.push({op:"replace",path:[],value:r[d].base}))}else t=D(e,r,n,[]);return n.revoke(),n.patches&&n.patchListener(n.patches,n.inversePatches),t!==f?t:void 0}function D(n,e,t,r){var i=e[d];if(!i)return Object.isFrozen(e)?e:j(n,e,t);if(i.scope!==t)return e;if(!i.modified)return F(n,i.base,!0),i.base;if(!i.finalized){if(i.finalized=!0,j(n,i.draft,t,r),n.onDelete&&i.type!==c.Set)if(n.useProxies){b(i.assigned,function(e,t){t||n.onDelete(i,e)})}else{var o=i.base,a=i.copy;b(o,function(e){E(a,e)||n.onDelete(i,e)})}n.onCopy&&n.onCopy(i),n.autoFreeze&&t.canAutoFreeze&&A(i.copy,!1),r&&t.patches&&function(e,t,n,r){switch(e.type){case c.ProxyObject:case c.ES5Object:case c.Map:return function(e,a,s,l){var u=e.base,c=e.copy;b(e.assigned,function(e,t){var n=x(u,e),r=x(c,e),i=t?E(u,e)?"replace":"add":"remove";if(n!==r||"replace"!=i){var o=a.concat(e);s.push("remove"==i?{op:i,path:o}:{op:i,path:o,value:r}),l.push("add"==i?{op:"remove",path:o}:"remove"==i?{op:"add",path:o,value:n}:{op:"replace",path:o,value:n})}})}(e,t,n,r);case c.ES5Array:case c.ProxyArray:return function(e,t,n,r){var i,o,a=e.base,s=e.assigned,l=e.copy;l||N();l.length=t.status}function a(t){try{t.dispatchEvent(new MouseEvent("click"))}catch(e){var n=document.createEvent("MouseEvents");n.initMouseEvent("click",!0,!0,window,0,0,0,80,20,!1,!1,!1,!1,0,null),t.dispatchEvent(n)}}var f="object"==typeof window&&window.window===window?window:"object"==typeof self&&self.self===self?self:"object"==typeof e&&e.global===e?e:void 0,s=f.saveAs||("object"!=typeof window||window!==f?function(){}:"download"in HTMLAnchorElement.prototype?function(e,t,n){var r=f.URL||f.webkitURL,i=document.createElement("a");t=t||e.name||"download",i.download=t,i.rel="noopener","string"==typeof e?(i.href=e,i.origin===location.origin?a(i):o(i.href)?c(e,t,n):a(i,i.target="_blank")):(i.href=r.createObjectURL(e),setTimeout(function(){r.revokeObjectURL(i.href)},4e4),setTimeout(function(){a(i)},0))}:"msSaveOrOpenBlob"in navigator?function(e,t,n){if(t=t||e.name||"download","string"!=typeof e)navigator.msSaveOrOpenBlob(i(e,n),t);else if(o(e))c(e,t,n);else{var r=document.createElement("a");r.href=e,r.target="_blank",setTimeout(function(){a(r)})}}:function(e,t,n,r){if(r=r||open("","_blank"),r&&(r.document.title=r.document.body.innerText="downloading..."),"string"==typeof e)return c(e,t,n);var i="application/octet-stream"===e.type,o=/constructor/i.test(f.HTMLElement)||f.safari,a=/CriOS\/[\d]+/.test(navigator.userAgent);if((a||i&&o)&&"object"==typeof FileReader){var s=new FileReader;s.onloadend=function(){var e=s.result;e=a?e:e.replace(/^data:[^;]*;/,"data:attachment/file;"),r?r.location.href=e:location=e,r=null},s.readAsDataURL(e)}else{var l=f.URL||f.webkitURL,u=l.createObjectURL(e);r?r.location=u:location.href=u,r=null,setTimeout(function(){l.revokeObjectURL(u)},4e4)}});f.saveAs=s.saveAs=s,true&&(l.exports=s)})?t.apply(i,n):t)||(l.exports=r)}).call(this,e(73))},function(e,t,n){var r=n(7),i=n(6).document,o=r(i)&&r(i.createElement);e.exports=function(e){return o?i.createElement(e):{}}},function(e,t,n){var r=n(6),i=n(20),o=n(48),a=n(138),s=n(15).f;e.exports=function(e){var t=i.Symbol||(i.Symbol=o?{}:r.Symbol||{});"_"==e.charAt(0)||e in t||s(t,e,{value:a.f(e)})}},function(e,t,n){var r=n(67)("keys"),i=n(54);e.exports=function(e){return r[e]||(r[e]=i(e))}},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t,n){var r=n(6).document;e.exports=r&&r.documentElement},function(e,t,n){"use strict";var d=n(12),p=n(43),h=n(77),v=n(69),m=n(17),g=n(68),i=Object.assign;e.exports=!i||n(8)(function(){var e={},t={},n=Symbol(),r="abcdefghijklmnopqrst";return e[n]=7,r.split("").forEach(function(e){t[e]=e}),7!=i({},e)[n]||Object.keys(i({},t)).join("")!=r})?function(e,t){for(var n=m(e),r=arguments.length,i=1,o=h.f,a=v.f;i>>=1)&&(t+=t))1&r&&(n+=t);return n}},function(e,t){e.exports=Math.sign||function(e){return 0==(e=+e)||e!=e?e:e<0?-1:1}},function(e,t){var n=Math.expm1;e.exports=!n||22025.465794806718=e.length?(this._t=void 0,i(1)):i(0,"keys"==t?n:"values"==t?e[n]:[n,e[n]])},"values"),o.Arguments=o.Array,r("keys"),r("values"),r("entries")},function(e,t){e.exports=function(e,t){return{value:t,done:!!e}}},function(e,t,n){"use strict";var r,i,a=n(71),s=RegExp.prototype.exec,l=String.prototype.replace,o=s,u="lastIndex",c=(r=/a/,i=/b*/g,s.call(r,"a"),s.call(i,"a"),0!==r[u]||0!==i[u]),f=void 0!==/()??/.exec("")[1];(c||f)&&(o=function(e){var t,n,r,i,o=this;return f&&(n=new RegExp("^"+o.source+"$(?!\\s)",a.call(o))),c&&(t=o[u]),r=s.call(o,e),c&&r&&(o[u]=o.global?r.index+r[0].length:t),f&&r&&1>1,c=23===t?A(2,-24)-A(2,-77):0,f=0,d=e<0||0===e&&1/e<0?1:0;for((e=_(e))!=e||e===C?(i=e!=e?1:0,r=l):(r=I(N(e)/L),e*(o=A(2,-r))<1&&(r--,o*=2),2<=(e+=1<=r+u?c/o:c*A(2,1-u))*o&&(r++,o/=2),l<=r+u?(i=0,r=l):1<=r+u?(i=(e*o-1)*A(2,t),r+=u):(i=e*A(2,u-1)*A(2,t),r=0));8<=t;a[f++]=255&i,i/=256,t-=8);for(r=r<>1,s=i-7,l=n-1,u=e[l--],c=127&u;for(u>>=7;0>=-s,s+=t;0>8&255]}function B(e){return[255&e,e>>8&255,e>>16&255,e>>24&255]}function V(e){return F(e,52,8)}function G(e){return F(e,23,4)}function q(e,t,n){v(e[w],t,{get:function(){return this[n]}})}function Y(e,t,n,r){var i=p(+n);if(i+t>e[D])throw O(E);var o=e[M]._b,a=i+e[j],s=o.slice(a,a+t);return r?s:s.reverse()}function X(e,t,n,r,i,o){var a=p(+n);if(a+t>e[D])throw O(E);for(var s=e[M]._b,l=a+e[j],u=r(+i),c=0;cJ;)(K=Q[J++])in x||s(x,K,k[K]);o||($.constructor=x)}var Z=new S(new x(2)),ee=S[w].setInt8;Z.setInt8(0,2147483648),Z.setInt8(1,2147483649),!Z.getInt8(0)&&Z.getInt8(1)||l(S[w],{setInt8:function(e,t){ee.call(this,e,t<<24>>24)},setUint8:function(e,t){ee.call(this,e,t<<24>>24)}},!0)}else x=function(e){c(this,x,y);var t=p(e);this._b=m.call(new Array(t),0),this[D]=t},S=function(e,t,n){c(this,S,b),c(e,x,b);var r=e[D],i=f(t);if(i<0||r>24},getUint8:function(e){return Y(this,1,e)[0]},getInt16:function(e,t){var n=Y(this,2,e,t);return(n[1]<<8|n[0])<<16>>16},getUint16:function(e,t){var n=Y(this,2,e,t);return n[1]<<8|n[0]},getInt32:function(e,t){return W(Y(this,4,e,t))},getUint32:function(e,t){return W(Y(this,4,e,t))>>>0},getFloat32:function(e,t){return H(Y(this,4,e,t),23,4)},getFloat64:function(e,t){return H(Y(this,8,e,t),52,8)},setInt8:function(e,t){X(this,1,e,U,t)},setUint8:function(e,t){X(this,1,e,U,t)},setInt16:function(e,t,n){X(this,2,e,z,t,n)},setUint16:function(e,t,n){X(this,2,e,z,t,n)},setInt32:function(e,t,n){X(this,4,e,B,t,n)},setUint32:function(e,t,n){X(this,4,e,B,t,n)},setFloat32:function(e,t,n){X(this,4,e,G,t,n)},setFloat64:function(e,t,n){X(this,8,e,V,t,n)}});g(x,y),g(S,b),s(S[w],a.VIEW,!0),t[y]=x,t[b]=S},function(e,t,n){var r=n(56),i=n(77),o=n(4),a=n(6).Reflect;e.exports=a&&a.ownKeys||function(e){var t=r.f(o(e)),n=i.f;return n?t.concat(n(e)):t}},function(e,t){e.exports=function(t,n){var r=n===Object(n)?function(e){return n[e]}:n;return function(e){return String(e).replace(t,r)}}},function(e,t,n){e.exports=n(414)()},function(e,t,n){"use strict";function s(e,t){for(var n=t&&t.plainObjects?Object.create(null):{},r=0;r>6]+u[128|63&a]:a<55296||57344<=a?i+=u[224|a>>12]+u[128|a>>6&63]+u[128|63&a]:(o+=1,a=65536+((1023&a)<<10|1023&r.charCodeAt(o)),i+=u[240|a>>18]+u[128|a>>12&63]+u[128|a>>6&63]+u[128|63&a])}return i},isBuffer:function(e){return!(!e||"object"!=typeof e)&&!!(e.constructor&&e.constructor.isBuffer&&e.constructor.isBuffer(e))},isRegExp:function(e){return"[object RegExp]"===Object.prototype.toString.call(e)},merge:function r(i,o,a){if(!o)return i;if("object"!=typeof o){if(c(i))i.push(o);else{if(!i||"object"!=typeof i)return[i,o];(a&&(a.plainObjects||a.allowPrototypes)||!l.call(Object.prototype,o))&&(i[o]=!0)}return i}if(!i||"object"!=typeof i)return[i].concat(o);var e=i;return c(i)&&!c(o)&&(e=s(i,a)),c(i)&&c(o)?(o.forEach(function(e,t){if(l.call(i,t)){var n=i[t];n&&"object"==typeof n&&e&&"object"==typeof e?i[t]=r(n,e,a):i.push(e)}else i[t]=e}),i):Object.keys(o).reduce(function(e,t){var n=o[t];return l.call(e,t)?e[t]=r(e[t],n,a):e[t]=n,e},e)}}},function(s,e,l){"use strict";(function(e){var n=l(30),r=l(428),t={"Content-Type":"application/x-www-form-urlencoded"};function i(e,t){!n.isUndefined(e)&&n.isUndefined(e["Content-Type"])&&(e["Content-Type"]=t)}var o,a={adapter:("undefined"!=typeof XMLHttpRequest?o=l(176):void 0!==e&&(o=l(176)),o),transformRequest:[function(e,t){return r(t,"Content-Type"),n.isFormData(e)||n.isArrayBuffer(e)||n.isBuffer(e)||n.isStream(e)||n.isFile(e)||n.isBlob(e)?e:n.isArrayBufferView(e)?e.buffer:n.isURLSearchParams(e)?(i(t,"application/x-www-form-urlencoded;charset=utf-8"),e.toString()):n.isObject(e)?(i(t,"application/json;charset=utf-8"),JSON.stringify(e)):e}],transformResponse:[function(e){if("string"==typeof e)try{e=JSON.parse(e)}catch(e){}return e}],timeout:0,xsrfCookieName:"XSRF-TOKEN",xsrfHeaderName:"X-XSRF-TOKEN",maxContentLength:-1,validateStatus:function(e){return 200<=e&&e<300}};a.headers={common:{Accept:"application/json, text/plain, */*"}},n.forEach(["delete","get","head"],function(e){a.headers[e]={}}),n.forEach(["post","put","patch"],function(e){a.headers[e]=n.merge(t)}),s.exports=a}).call(this,l(128))},function(e,t){var n,r,i=e.exports={};function o(){throw new Error("setTimeout has not been defined")}function a(){throw new Error("clearTimeout has not been defined")}function s(t){if(n===setTimeout)return setTimeout(t,0);if((n===o||!n)&&setTimeout)return n=setTimeout,setTimeout(t,0);try{return n(t,0)}catch(e){try{return n.call(null,t,0)}catch(e){return n.call(this,t,0)}}}!function(){try{n="function"==typeof setTimeout?setTimeout:o}catch(e){n=o}try{r="function"==typeof clearTimeout?clearTimeout:a}catch(e){r=a}}();var l,u=[],c=!1,f=-1;function d(){c&&l&&(c=!1,l.length?u=l.concat(u):f=-1,u.length&&p())}function p(){if(!c){var e=s(d);c=!0;for(var t=u.length;t;){for(l=u,u=[];++fi;)a(r,n=t[i++])&&(~l(o,n)||o.push(n));return o}},function(e,t,n){var a=n(15),s=n(4),l=n(43);e.exports=n(12)?Object.defineProperties:function(e,t){s(e);for(var n,r=l(t),i=r.length,o=0;o>>0||(a.test(n)?16:10))}:r},function(e,t,n){var r=n(6).parseFloat,i=n(64).trim;e.exports=1/r(n(104)+"-0")!=-1/0?function(e){var t=i(String(e),3),n=r(t);return 0===n&&"-"==t.charAt(0)?-0:n}:r},function(e,t,n){var r=n(32);e.exports=function(e,t){if("number"!=typeof e&&"Number"!=r(e))throw TypeError(t);return+e}},function(e,t,n){var r=n(7),i=Math.floor;e.exports=function(e){return!r(e)&&isFinite(e)&&i(e)===e}},function(e,t){e.exports=Math.log1p||function(e){return-1e-8<(e=+e)&&e<1e-8?e-e*e/2:Math.log(1+e)}},function(e,t,n){var o=n(107),r=Math.pow,a=r(2,-52),s=r(2,-23),l=r(2,127)*(2-s),u=r(2,-126);e.exports=Math.fround||function(e){var t,n,r=Math.abs(e),i=o(e);return rl&&(u=u.slice(0,l)),r?u+i:i+u}},function(e,t,n){var l=n(12),u=n(43),c=n(21),f=n(69).f;e.exports=function(s){return function(e){for(var t,n=c(e),r=u(n),i=r.length,o=0,a=[];oe.length)&&(t=e.length);for(var n=0,r=new Array(t);n'+e+""}).then(function(e){return''+e+""}).then(function(e){return"data:image/svg+xml;charset=utf-8,"+e});var t,n,r})}function v(n,r){return p(n,r).then(h.makeImage).then(h.delay(100)).then(function(e){var t=function(e){var t=document.createElement("canvas");if(t.width=r.width||h.width(e),t.height=r.height||h.height(e),r.bgcolor){var n=t.getContext("2d");n.fillStyle=r.bgcolor,n.fillRect(0,0,t.width,t.height)}return t}(n);return t.getContext("2d").drawImage(e,0,0),t})}function m(n){return t.resolveAll().then(function(e){var t=document.createElement("style");return n.appendChild(t),t.appendChild(document.createTextNode(e)),n})}function g(e){return u.inlineAll(e).then(function(){return e})}y.exports=d}()},function(e,n,t){"use strict";Object.defineProperty(n,"__esModule",{value:!0}),function(e){for(var t in e)n.hasOwnProperty(t)||(n[t]=e[t])}(t(475))},function(module,exports,__webpack_require__){(function(process,global){var __WEBPACK_AMD_DEFINE_RESULT__;!function(){"use strict";var ERROR="input is invalid type",WINDOW="object"==typeof window,root=WINDOW?window:{};root.JS_SHA256_NO_WINDOW&&(WINDOW=!1);var WEB_WORKER=!WINDOW&&"object"==typeof self,NODE_JS=!root.JS_SHA256_NO_NODE_JS&&"object"==typeof process&&process.versions&&process.versions.node;NODE_JS?root=global:WEB_WORKER&&(root=self);var COMMON_JS=!root.JS_SHA256_NO_COMMON_JS&&"object"==typeof module&&module.exports,AMD=__webpack_require__(476),ARRAY_BUFFER=!root.JS_SHA256_NO_ARRAY_BUFFER&&"undefined"!=typeof ArrayBuffer,HEX_CHARS="0123456789abcdef".split(""),EXTRA=[-2147483648,8388608,32768,128],SHIFT=[24,16,8,0],K=[1116352408,1899447441,3049323471,3921009573,961987163,1508970993,2453635748,2870763221,3624381080,310598401,607225278,1426881987,1925078388,2162078206,2614888103,3248222580,3835390401,4022224774,264347078,604807628,770255983,1249150122,1555081692,1996064986,2554220882,2821834349,2952996808,3210313671,3336571891,3584528711,113926993,338241895,666307205,773529912,1294757372,1396182291,1695183700,1986661051,2177026350,2456956037,2730485921,2820302411,3259730800,3345764771,3516065817,3600352804,4094571909,275423344,430227734,506948616,659060556,883997877,958139571,1322822218,1537002063,1747873779,1955562222,2024104815,2227730452,2361852424,2428436474,2756734187,3204031479,3329325298],OUTPUT_TYPES=["hex","array","digest","arrayBuffer"],blocks=[];!root.JS_SHA256_NO_NODE_JS&&Array.isArray||(Array.isArray=function(e){return"[object Array]"===Object.prototype.toString.call(e)}),!ARRAY_BUFFER||!root.JS_SHA256_NO_ARRAY_BUFFER_IS_VIEW&&ArrayBuffer.isView||(ArrayBuffer.isView=function(e){return"object"==typeof e&&e.buffer&&e.buffer.constructor===ArrayBuffer});var createOutputMethod=function(t,n){return function(e){return new Sha256(n,!0).update(e)[t]()}},createMethod=function(e){var t=createOutputMethod("hex",e);NODE_JS&&(t=nodeWrap(t,e)),t.create=function(){return new Sha256(e)},t.update=function(e){return t.create().update(e)};for(var n=0;n>6:(o<55296||57344<=o?a[l++]=224|o>>12:(o=65536+((1023&o)<<10|1023&e.charCodeAt(++r)),a[l++]=240|o>>18,a[l++]=128|o>>12&63),a[l++]=128|o>>6&63),a[l++]=128|63&o);e=a}else{if("object"!=i)throw new Error(ERROR);if(null===e)throw new Error(ERROR);if(ARRAY_BUFFER&&e.constructor===ArrayBuffer)e=new Uint8Array(e);else if(!(Array.isArray(e)||ARRAY_BUFFER&&ArrayBuffer.isView(e)))throw new Error(ERROR)}64>2]|=e[o]<>2]|=r<>2]|=(192|r>>6)<>2]|=(224|r>>12)<>2]|=(240|r>>18)<>2]|=(128|r>>12&63)<>2]|=(128|r>>6&63)<>2]|=(128|63&r)<>2]|=EXTRA[3&t],this.block=e[16],56<=t&&(this.hashed||this.hash(),e[0]=this.block,e[16]=e[1]=e[2]=e[3]=e[4]=e[5]=e[6]=e[7]=e[8]=e[9]=e[10]=e[11]=e[12]=e[13]=e[14]=e[15]=0),e[14]=this.hBytes<<3|this.bytes>>>29,e[15]=this.bytes<<3,this.hash()}},Sha256.prototype.hash=function(){var e,t,n,r,i,o,a,s,l,u=this.h0,c=this.h1,f=this.h2,d=this.h3,p=this.h4,h=this.h5,v=this.h6,m=this.h7,g=this.blocks;for(e=16;e<64;++e)t=((i=g[e-15])>>>7|i<<25)^(i>>>18|i<<14)^i>>>3,n=((i=g[e-2])>>>17|i<<15)^(i>>>19|i<<13)^i>>>10,g[e]=g[e-16]+t+g[e-7]+n<<0;for(l=c&f,e=0;e<64;e+=4)this.first?(d=this.is224?(o=300032,m=(i=g[0]-1413257819)-150054599<<0,i+24177077<<0):(o=704751109,m=(i=g[0]-210244248)-1521486534<<0,i+143694565<<0),this.first=!1):(t=(u>>>2|u<<30)^(u>>>13|u<<19)^(u>>>22|u<<10),r=(o=u&c)^u&f^l,m=d+(i=m+(n=(p>>>6|p<<26)^(p>>>11|p<<21)^(p>>>25|p<<7))+(p&h^~p&v)+K[e]+g[e])<<0,d=i+(t+r)<<0),t=(d>>>2|d<<30)^(d>>>13|d<<19)^(d>>>22|d<<10),r=(a=d&u)^d&c^o,v=f+(i=v+(n=(m>>>6|m<<26)^(m>>>11|m<<21)^(m>>>25|m<<7))+(m&p^~m&h)+K[e+1]+g[e+1])<<0,t=((f=i+(t+r)<<0)>>>2|f<<30)^(f>>>13|f<<19)^(f>>>22|f<<10),r=(s=f&d)^f&u^a,h=c+(i=h+(n=(v>>>6|v<<26)^(v>>>11|v<<21)^(v>>>25|v<<7))+(v&m^~v&p)+K[e+2]+g[e+2])<<0,t=((c=i+(t+r)<<0)>>>2|c<<30)^(c>>>13|c<<19)^(c>>>22|c<<10),r=(l=c&f)^c&d^s,p=u+(i=p+(n=(h>>>6|h<<26)^(h>>>11|h<<21)^(h>>>25|h<<7))+(h&v^~h&m)+K[e+3]+g[e+3])<<0,u=i+(t+r)<<0;this.h0=this.h0+u<<0,this.h1=this.h1+c<<0,this.h2=this.h2+f<<0,this.h3=this.h3+d<<0,this.h4=this.h4+p<<0,this.h5=this.h5+h<<0,this.h6=this.h6+v<<0,this.h7=this.h7+m<<0},Sha256.prototype.hex=function(){this.finalize();var e=this.h0,t=this.h1,n=this.h2,r=this.h3,i=this.h4,o=this.h5,a=this.h6,s=this.h7,l=HEX_CHARS[e>>28&15]+HEX_CHARS[e>>24&15]+HEX_CHARS[e>>20&15]+HEX_CHARS[e>>16&15]+HEX_CHARS[e>>12&15]+HEX_CHARS[e>>8&15]+HEX_CHARS[e>>4&15]+HEX_CHARS[15&e]+HEX_CHARS[t>>28&15]+HEX_CHARS[t>>24&15]+HEX_CHARS[t>>20&15]+HEX_CHARS[t>>16&15]+HEX_CHARS[t>>12&15]+HEX_CHARS[t>>8&15]+HEX_CHARS[t>>4&15]+HEX_CHARS[15&t]+HEX_CHARS[n>>28&15]+HEX_CHARS[n>>24&15]+HEX_CHARS[n>>20&15]+HEX_CHARS[n>>16&15]+HEX_CHARS[n>>12&15]+HEX_CHARS[n>>8&15]+HEX_CHARS[n>>4&15]+HEX_CHARS[15&n]+HEX_CHARS[r>>28&15]+HEX_CHARS[r>>24&15]+HEX_CHARS[r>>20&15]+HEX_CHARS[r>>16&15]+HEX_CHARS[r>>12&15]+HEX_CHARS[r>>8&15]+HEX_CHARS[r>>4&15]+HEX_CHARS[15&r]+HEX_CHARS[i>>28&15]+HEX_CHARS[i>>24&15]+HEX_CHARS[i>>20&15]+HEX_CHARS[i>>16&15]+HEX_CHARS[i>>12&15]+HEX_CHARS[i>>8&15]+HEX_CHARS[i>>4&15]+HEX_CHARS[15&i]+HEX_CHARS[o>>28&15]+HEX_CHARS[o>>24&15]+HEX_CHARS[o>>20&15]+HEX_CHARS[o>>16&15]+HEX_CHARS[o>>12&15]+HEX_CHARS[o>>8&15]+HEX_CHARS[o>>4&15]+HEX_CHARS[15&o]+HEX_CHARS[a>>28&15]+HEX_CHARS[a>>24&15]+HEX_CHARS[a>>20&15]+HEX_CHARS[a>>16&15]+HEX_CHARS[a>>12&15]+HEX_CHARS[a>>8&15]+HEX_CHARS[a>>4&15]+HEX_CHARS[15&a];return this.is224||(l+=HEX_CHARS[s>>28&15]+HEX_CHARS[s>>24&15]+HEX_CHARS[s>>20&15]+HEX_CHARS[s>>16&15]+HEX_CHARS[s>>12&15]+HEX_CHARS[s>>8&15]+HEX_CHARS[s>>4&15]+HEX_CHARS[15&s]),l},Sha256.prototype.toString=Sha256.prototype.hex,Sha256.prototype.digest=function(){this.finalize();var e=this.h0,t=this.h1,n=this.h2,r=this.h3,i=this.h4,o=this.h5,a=this.h6,s=this.h7,l=[e>>24&255,e>>16&255,e>>8&255,255&e,t>>24&255,t>>16&255,t>>8&255,255&t,n>>24&255,n>>16&255,n>>8&255,255&n,r>>24&255,r>>16&255,r>>8&255,255&r,i>>24&255,i>>16&255,i>>8&255,255&i,o>>24&255,o>>16&255,o>>8&255,255&o,a>>24&255,a>>16&255,a>>8&255,255&a];return this.is224||l.push(s>>24&255,s>>16&255,s>>8&255,255&s),l},Sha256.prototype.array=Sha256.prototype.digest,Sha256.prototype.arrayBuffer=function(){this.finalize();var e=new ArrayBuffer(this.is224?28:32),t=new DataView(e);return t.setUint32(0,this.h0),t.setUint32(4,this.h1),t.setUint32(8,this.h2),t.setUint32(12,this.h3),t.setUint32(16,this.h4),t.setUint32(20,this.h5),t.setUint32(24,this.h6),this.is224||t.setUint32(28,this.h7),e},HmacSha256.prototype=new Sha256,HmacSha256.prototype.finalize=function(){if(Sha256.prototype.finalize.call(this),this.inner){this.inner=!1;var e=this.array();Sha256.call(this,this.is224,this.sharedMemory),this.update(this.oKeyPad),this.update(e),Sha256.prototype.finalize.call(this)}};var exports=createMethod();exports.sha256=exports,exports.sha224=createMethod(!0),exports.sha256.hmac=createHmacMethod(),exports.sha224.hmac=createHmacMethod(!0),COMMON_JS?module.exports=exports:(root.sha256=exports.sha256,root.sha224=exports.sha224,AMD&&(__WEBPACK_AMD_DEFINE_RESULT__=function(){return exports}.call(exports,__webpack_require__,exports,module),void 0===__WEBPACK_AMD_DEFINE_RESULT__||(module.exports=__WEBPACK_AMD_DEFINE_RESULT__)))}()}).call(this,__webpack_require__(128),__webpack_require__(73))},function(e,t,n){e.exports=function(){"use strict";var e=navigator.userAgent,t=navigator.platform,v=/gecko\/\d/i.test(e),n=/MSIE \d/.test(e),r=/Trident\/(?:[7-9]|\d{2,})\..*rv:(\d+)/.exec(e),i=/Edge\/(\d+)/.exec(e),E=n||r||i,x=E&&(n?document.documentMode||6:+(i||r)[1]),m=!i&&/WebKit\//.test(e),o=m&&/Qt\/\d+\.\d+/.test(e),a=!i&&/Chrome\//.test(e),g=/Opera\//.test(e),s=/Apple Computer/.test(navigator.vendor),l=/Mac OS X 1\d\D([8-9]|\d\d)\D/.test(e),u=/PhantomJS/.test(e),c=!i&&/AppleWebKit/.test(e)&&/Mobile\/\w+/.test(e),f=/Android/.test(e),d=c||f||/webOS|BlackBerry|Opera Mini|Opera Mobi|IEMobile/i.test(e),y=c||/Mac/.test(t),p=/\bCrOS\b/.test(e),h=/win/i.test(t),b=g&&e.match(/Version\/(\d*\.\d*)/);if(b){b=Number(b[1])}if(b&&b>=15){g=false;m=true}var w=y&&(o||g&&(b==null||b<12.11)),S=v||E&&x>=9;function T(e){return new RegExp("(^|\\s)"+e+"(?:$|\\s)\\s*")}var O=function(e,t){var n=e.className;var r=T(t).exec(n);if(r){var i=n.slice(r.index+r[0].length);e.className=n.slice(0,r.index)+(i?r[1]+i:"")}},C;function k(e){for(var t=e.childNodes.length;t>0;--t){e.removeChild(e.firstChild)}return e}function _(e,t){return k(e).appendChild(t)}function N(e,t,n,r){var i=document.createElement(e);if(n){i.className=n}if(r){i.style.cssText=r}if(typeof t=="string"){i.appendChild(document.createTextNode(t))}else if(t){for(var o=0;o=t){return a+(t-o)}a+=s-o;a+=n-a%n;o=s+1}}var H=function(){this.id=null;this.f=null;this.time=0;this.handler=D(this.onTimeout,this)};function W(e,t){for(var n=0;n=t){return r+Math.min(a,t-i)}i+=o-r;i+=n-i%n;r=o+1;if(i>=t){return r}}}var Y=[""];function X(e){while(Y.length<=e){Y.push(K(Y)+" ")}return Y[e]}function K(e){return e[e.length-1]}function $(e,t){var n=[];for(var r=0;r"€"&&(e.toUpperCase()!=e.toLowerCase()||ee.test(e))}function ne(e,t){if(!t){return te(e)}if(t.source.indexOf("\\w")>-1&&te(e)){return true}return t.test(e)}function re(e){for(var t in e){if(e.hasOwnProperty(t)&&e[t]){return false}}return true}var ie=/[\u0300-\u036f\u0483-\u0489\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u064b-\u065e\u0670\u06d6-\u06dc\u06de-\u06e4\u06e7\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0900-\u0902\u093c\u0941-\u0948\u094d\u0951-\u0955\u0962\u0963\u0981\u09bc\u09be\u09c1-\u09c4\u09cd\u09d7\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a70\u0a71\u0a75\u0a81\u0a82\u0abc\u0ac1-\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3e\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b57\u0b62\u0b63\u0b82\u0bbe\u0bc0\u0bcd\u0bd7\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62\u0c63\u0cbc\u0cbf\u0cc2\u0cc6\u0ccc\u0ccd\u0cd5\u0cd6\u0ce2\u0ce3\u0d3e\u0d41-\u0d44\u0d4d\u0d57\u0d62\u0d63\u0dca\u0dcf\u0dd2-\u0dd4\u0dd6\u0ddf\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb\u0ebc\u0ec8-\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86\u0f87\u0f90-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039\u103a\u103d\u103e\u1058\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085\u1086\u108d\u109d\u135f\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927\u1928\u1932\u1939-\u193b\u1a17\u1a18\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80\u1b81\u1ba2-\u1ba5\u1ba8\u1ba9\u1c2c-\u1c33\u1c36\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1dc0-\u1de6\u1dfd-\u1dff\u200c\u200d\u20d0-\u20f0\u2cef-\u2cf1\u2de0-\u2dff\u302a-\u302f\u3099\u309a\ua66f-\ua672\ua67c\ua67d\ua6f0\ua6f1\ua802\ua806\ua80b\ua825\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31\uaa32\uaa35\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7\uaab8\uaabe\uaabf\uaac1\uabe5\uabe8\uabed\udc00-\udfff\ufb1e\ufe00-\ufe0f\ufe20-\ufe26\uff9e\uff9f]/;function oe(e){return e.charCodeAt(0)>=768&&ie.test(e)}function ae(e,t,n){while((n<0?t>0:tn?-1:1;for(;;){if(t==n){return t}var i=(t+n)/2,o=r<0?Math.ceil(i):Math.floor(i);if(o==t){return e(o)?t:n}if(e(o)){n=o}else{t=o+r}}}function le(e,t,n,r){if(!e){return r(t,n,"ltr",0)}var i=false;for(var o=0;ot||t==n&&a.to==t){r(Math.max(a.from,t),Math.min(a.to,n),a.level==1?"rtl":"ltr",o);i=true}}if(!i){r(t,n,"ltr")}}var ue=null;function ce(e,t,n){var r;ue=null;for(var i=0;it){return i}if(o.to==t){if(o.from!=o.to&&n=="before"){r=i}else{ue=i}}if(o.from==t){if(o.from!=o.to&&n!="before"){r=i}else{ue=i}}}return r!=null?r:ue}var fe=function(){var t="bbbbbbbbbtstwsbbbbbbbbbbbbbbssstwNN%%%NNNNNN,N,N1111111111NNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNbbbbbbsbbbbbbbbbbbbbbbbbbbbbbbbbb,N%%%%NNNNLNNNNN%%11NLNNN1LNNNNNLLLLLLLLLLLLLLLLLLLLLLLNLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLN";var n="nnnnnnNNr%%r,rNNmmmmmmmmmmmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmmmmmmmmmmmmmmmnnnnnnnnnn%nnrrrmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmnNmmmmmmrrmmNmmmmrr1111111111";function F(e){if(e<=247){return t.charAt(e)}else if(1424<=e&&e<=1524){return"R"}else if(1536<=e&&e<=1785){return n.charAt(e-1536)}else if(1774<=e&&e<=2220){return"r"}else if(8192<=e&&e<=8203){return"w"}else if(e==8204){return"b"}else{return"L"}}var H=/[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac]/;var W=/[stwN]/,U=/[LRr]/,z=/[Lb1n]/,B=/[1n]/;function V(e,t,n){this.level=e;this.from=t;this.to=n}return function(e,t){var n=t=="ltr"?"L":"R";if(e.length==0||t=="ltr"&&!H.test(e)){return false}var r=e.length,i=[];for(var o=0;o-1){r[t]=i.slice(0,o).concat(i.slice(o+1))}}}}function ge(e,t){var n=ve(e,t);if(!n.length){return}var r=Array.prototype.slice.call(arguments,2);for(var i=0;i0}function Ee(e){e.prototype.on=function(e,t){he(this,e,t)};e.prototype.off=function(e,t){me(this,e,t)}}function xe(e){if(e.preventDefault){e.preventDefault()}else{e.returnValue=false}}function Se(e){if(e.stopPropagation){e.stopPropagation()}else{e.cancelBubble=true}}function Te(e){return e.defaultPrevented!=null?e.defaultPrevented:e.returnValue==false}function Oe(e){xe(e);Se(e)}function Ce(e){return e.target||e.srcElement}function ke(e){var t=e.which;if(t==null){if(e.button&1){t=1}else if(e.button&2){t=3}else if(e.button&4){t=2}}if(y&&e.ctrlKey&&t==1){t=3}return t}var _e=function(){if(E&&x<9){return false}var e=N("div");return"draggable"in e||"dragDrop"in e}(),Ae,Ie;function Ne(e){if(Ae==null){var t=N("span","​");_(e,N("span",[t,document.createTextNode("x")]));if(e.firstChild.offsetHeight!=0){Ae=t.offsetWidth<=1&&t.offsetHeight>2&&!(E&&x<8)}}var n=Ae?N("span","​"):N("span"," ",null,"display: inline-block; width: 1px; margin-right: -1px");n.setAttribute("cm-text","");return n}function Le(e){if(Ie!=null){return Ie}var t=_(e,document.createTextNode("AخA"));var n=C(t,0,1).getBoundingClientRect();var r=C(t,1,2).getBoundingClientRect();k(e);if(!n||n.left==n.right){return false}return Ie=r.right-n.right<3}var Re="\n\nb".split(/\n/).length!=3?function(e){var t=0,n=[],r=e.length;while(t<=r){var i=e.indexOf("\n",t);if(i==-1){i=e.length}var o=e.slice(t,e.charAt(i-1)=="\r"?i-1:i);var a=o.indexOf("\r");if(a!=-1){n.push(o.slice(0,a));t+=a+1}else{n.push(o);t=i+1}}return n}:function(e){return e.split(/\r\n?|\n/)},Pe=window.getSelection?function(e){try{return e.selectionStart!=e.selectionEnd}catch(e){return false}}:function(e){var t;try{t=e.ownerDocument.selection.createRange()}catch(e){}if(!t||t.parentElement()!=e){return false}return t.compareEndPoints("StartToEnd",t)!=0},Me=function(){var e=N("div");if("oncopy"in e){return true}e.setAttribute("oncopy","return;");return typeof e.oncopy=="function"}(),De=null;function je(e){if(De!=null){return De}var t=_(e,N("span","x"));var n=t.getBoundingClientRect();var r=C(t,0,1).getBoundingClientRect();return De=Math.abs(n.left-r.left)>1}var Fe={},He={};function We(e,t){if(2=e.size){throw new Error("There is no line "+(t+e.first)+" in the document.")}var n=e;while(!n.lines){for(var r=0;;++r){var i=n.children[r],o=i.chunkSize();if(t=e.first&&tn){return it(n,$e(e,n).text.length)}return dt(t,$e(e,t.line).text.length)}function dt(e,t){var n=e.ch;if(n==null||n>t){return it(e.line,t)}else if(n<0){return it(e.line,0)}else{return e}}function pt(e,t){var n=[];for(var r=0;r=this.string.length},Ke.prototype.sol=function(){return this.pos==this.lineStart},Ke.prototype.peek=function(){return this.string.charAt(this.pos)||undefined},Ke.prototype.next=function(){if(this.post},Ke.prototype.eatSpace=function(){var e=this.pos;while(/[\s\u00a0]/.test(this.string.charAt(this.pos))){++this.pos}return this.pos>e},Ke.prototype.skipToEnd=function(){this.pos=this.string.length},Ke.prototype.skipTo=function(e){var t=this.string.indexOf(e,this.pos);if(t>-1){this.pos=t;return true}},Ke.prototype.backUp=function(e){this.pos-=e},Ke.prototype.column=function(){if(this.lastColumnPos0){return null}if(o&&t!==false){this.pos+=o[0].length}return o}},Ke.prototype.current=function(){return this.string.slice(this.start,this.pos)},Ke.prototype.hideFirstChars=function(e,t){this.lineStart+=e;try{return t()}finally{this.lineStart-=e}},Ke.prototype.lookAhead=function(e){var t=this.lineOracle;return t&&t.lookAhead(e)},Ke.prototype.baseToken=function(){var e=this.lineOracle;return e&&e.baseToken(this.pos)};var ht=function(e,t){this.state=e;this.lookAhead=t},vt=function(e,t,n,r){this.state=t;this.doc=e;this.line=n;this.maxLookAhead=r||0;this.baseTokens=null;this.baseTokenPos=1};function mt(t,n,r,e){var l=[t.state.modeGen],i={};Ot(t,n.text,t.doc.mode,r,function(e,t){return l.push(e,t)},i,e);var u=r.state;var o=function(e){r.baseTokens=l;var o=t.state.overlays[e],a=1,s=0;r.state=true;Ot(t,n.text,o.mode,r,function(e,t){var n=a;while(se){l.splice(a,1,e,l[a+1],r)}a+=2;s=Math.min(e,r)}if(!t){return}if(o.opaque){l.splice(n,a-n,e,"overlay "+t);a=n+2}else{for(;ne.options.maxHighlightLength&&qe(e.doc.mode,r.state);var o=mt(e,t,r);if(i){r.state=i}t.stateAfter=r.save(!i);t.styles=o.styles;if(o.classes){t.styleClasses=o.classes}else if(t.styleClasses){t.styleClasses=null}if(n===e.doc.highlightFrontier){e.doc.modeFrontier=Math.max(e.doc.modeFrontier,++e.doc.highlightFrontier)}}return t.styles}function yt(n,r,e){var t=n.doc,i=n.display;if(!t.mode.startState){return new vt(t,true,r)}var o=Ct(n,r,e);var a=o>t.first&&$e(t,o-1).stateAfter;var s=a?vt.fromSaved(t,a,o):new vt(t,Xe(t.mode),o);t.iter(o,r,function(e){bt(n,e.text,s);var t=s.line;e.stateAfter=t==r-1||t%5==0||t>=i.viewFrom&&tt.start){return o}}throw new Error("Mode "+e.name+" failed to advance stream.")}vt.prototype.lookAhead=function(e){var t=this.doc.getLine(this.line+e);if(t!=null&&e>this.maxLookAhead){this.maxLookAhead=e}return t},vt.prototype.baseToken=function(e){if(!this.baseTokens){return null}while(this.baseTokens[this.baseTokenPos]<=e){this.baseTokenPos+=2}var t=this.baseTokens[this.baseTokenPos+1];return{type:t&&t.replace(/( |^)overlay .*/,""),size:this.baseTokens[this.baseTokenPos]-e}},vt.prototype.nextLine=function(){this.line++;if(this.maxLookAhead>0){this.maxLookAhead--}},vt.fromSaved=function(e,t,n){if(t instanceof ht){return new vt(e,qe(e.mode,t.state),n,t.lookAhead)}else{return new vt(e,qe(e.mode,t),n)}},vt.prototype.save=function(e){var t=e!==false?qe(this.doc.mode,this.state):this.state;return this.maxLookAhead>0?new ht(t,this.maxLookAhead):t};var xt=function(e,t,n){this.start=e.start;this.end=e.pos;this.string=e.current();this.type=t||null;this.state=n};function St(e,t,n,r){var i=e.doc,o=i.mode,a;t=ft(i,t);var s=$e(i,t.line),l=yt(e,t.line,n);var u=new Ke(s.text,e.options.tabSize,l),c;if(r){c=[]}while((r||u.pose.options.maxHighlightLength){s=false;if(a){bt(e,t,r,c.pos)}c.pos=t.length;f=null}else{f=Tt(Et(n,c,r.state,d),o)}if(d){var p=d[0].name;if(p){f="m-"+(f?p+" "+f:p)}}if(!s||u!=f){while(la;--s){if(s<=o.first){return o.first}var l=$e(o,s-1),u=l.stateAfter;if(u&&(!n||s+(u instanceof ht?u.lookAhead:0)<=o.modeFrontier)){return s}var c=F(l.text,null,e.options.tabSize);if(i==null||r>c){i=s-1;r=c}}return i}function kt(e,t){e.modeFrontier=Math.min(e.modeFrontier,t);if(e.highlightFrontiern;r--){var i=$e(e,r).stateAfter;if(i&&(!(i instanceof ht)||r+i.lookAhead=t:o.to>t);(r||(r=[])).push(new Lt(a,o.from,l?null:o.to))}}}return r}function jt(e,t,n){var r;if(e){for(var i=0;i=t:o.to>t);if(s||o.from==t&&a.type=="bookmark"&&(!n||o.marker.insertLeft)){var l=o.from==null||(a.inclusiveLeft?o.from<=t:o.from0&&s){for(var w=0;w0){continue}var c=[l,1],f=ot(u.from,s.from),d=ot(u.to,s.to);if(f<0||!a.inclusiveLeft&&!f){c.push({from:u.from,to:s.from})}if(d>0||!a.inclusiveRight&&!d){c.push({from:s.to,to:u.to})}i.splice.apply(i,c);l+=c.length-3}}return i}function Ut(e){var t=e.markedSpans;if(!t){return}for(var n=0;nt)&&(!r||Gt(r,o.marker)<0)){r=o.marker}}}return r}function $t(e,t,n,r,i){var o=$e(e,t);var a=At&&o.markedSpans;if(a){for(var s=0;s=0&&f<=0||c<=0&&f>=0){continue}if(c<=0&&(l.marker.inclusiveRight&&i.inclusiveLeft?ot(u.to,n)>=0:ot(u.to,n)>0)||c>=0&&(l.marker.inclusiveRight&&i.inclusiveLeft?ot(u.from,r)<=0:ot(u.from,r)<0)){return true}}}}function Qt(e){var t;while(t=Yt(e)){e=t.find(-1,true).line}return e}function Jt(e){var t;while(t=Xt(e)){e=t.find(1,true).line}return e}function Zt(e){var t,n;while(t=Xt(e)){e=t.find(1,true).line;(n||(n=[])).push(e)}return n}function en(e,t){var n=$e(e,t),r=Qt(n);if(n==r){return t}return et(r)}function tn(e,t){if(t>e.lastLine()){return t}var n=$e(e,t),r;if(!nn(e,n)){return t}while(r=Xt(n)){n=r.find(1,true).line}return et(n)+1}function nn(e,t){var n=At&&t.markedSpans;if(n){for(var r=void 0,i=0;in.maxLineLength){n.maxLineLength=t;n.maxLine=e}})}var ln=function(e,t,n){this.text=e;zt(this,t);this.height=n?n(this):1};function un(e,t,n,r){e.text=t;if(e.stateAfter){e.stateAfter=null}if(e.styles){e.styles=null}if(e.order!=null){e.order=null}Ut(e);zt(e,n);var i=r?r(e):1;if(i!=e.height){Ze(e,i)}}function cn(e){e.parent=null;Ut(e)}ln.prototype.lineNo=function(){return et(this)},Ee(ln);var fn={},dn={};function pn(e,t){if(!e||/^\s*$/.test(e)){return null}var n=t.addModeClass?dn:fn;return n[e]||(n[e]=e.replace(/\S+/g,"cm-$&"))}function hn(e,t){var n=A("span",null,null,m?"padding-right: .1px":null);var r={pre:A("pre",[n],"CodeMirror-line"),content:n,col:0,pos:0,cm:e,trailingSpace:false,splitSpaces:e.getOption("lineWrapping")};t.measure={};for(var i=0;i<=(t.rest?t.rest.length:0);i++){var o=i?t.rest[i-1]:t.line,a=void 0;r.pos=0;r.addToken=mn;if(Le(e.display.measure)&&(a=de(o,e.doc.direction))){r.addToken=yn(r.addToken,a)}r.map=[];var s=t!=e.display.externalMeasured&&et(o);wn(o,r,gt(e,o,s));if(o.styleClasses){if(o.styleClasses.bgClass){r.bgClass=P(o.styleClasses.bgClass,r.bgClass||"")}if(o.styleClasses.textClass){r.textClass=P(o.styleClasses.textClass,r.textClass||"")}}if(r.map.length==0){r.map.push(0,0,r.content.appendChild(Ne(e.display.measure)))}if(i==0){t.measure.map=r.map;t.measure.cache={}}else{(t.measure.maps||(t.measure.maps=[])).push(r.map);(t.measure.caches||(t.measure.caches=[])).push({})}}if(m){var l=r.content.lastChild;if(/\bcm-tab\b/.test(l.className)||l.querySelector&&l.querySelector(".cm-tab")){r.content.className="cm-tab-wrap-hack"}}ge(e,"renderLine",e,t.line,r.pre);if(r.pre.className){r.textClass=P(r.pre.className,r.textClass||"")}return r}function vn(e){var t=N("span","•","cm-invalidchar");t.title="\\u"+e.charCodeAt(0).toString(16);t.setAttribute("aria-label",t.title);return t}function mn(e,t,n,r,i,o,a){if(!t){return}var s=e.splitSpaces?gn(t,e.trailingSpace):t;var l=e.cm.state.specialChars,u=false;var c;if(!l.test(t)){e.col+=t.length;c=document.createTextNode(s);e.map.push(e.pos,e.pos+t.length,c);if(E&&x<9){u=true}e.pos+=t.length}else{c=document.createDocumentFragment();var f=0;while(true){l.lastIndex=f;var d=l.exec(t);var p=d?d.index-f:t.length-f;if(p){var h=document.createTextNode(s.slice(f,f+p));if(E&&x<9){c.appendChild(N("span",[h]))}else{c.appendChild(h)}e.map.push(e.pos,e.pos+p,h);e.col+=p;e.pos+=p}if(!d){break}f+=p+1;var v=void 0;if(d[0]=="\t"){var m=e.cm.options.tabSize,g=m-e.col%m;v=c.appendChild(N("span",X(g),"cm-tab"));v.setAttribute("role","presentation");v.setAttribute("cm-text","\t");e.col+=g}else if(d[0]=="\r"||d[0]=="\n"){v=c.appendChild(N("span",d[0]=="\r"?"␍":"␤","cm-invalidchar"));v.setAttribute("cm-text",d[0]);e.col+=1}else{v=e.cm.options.specialCharPlaceholder(d[0]);v.setAttribute("cm-text",d[0]);if(E&&x<9){c.appendChild(N("span",[v]))}else{c.appendChild(v)}e.col+=1}e.map.push(e.pos,e.pos+1,v);e.pos++}}e.trailingSpace=s.charCodeAt(t.length-1)==32;if(n||r||i||u||o){var y=n||"";if(r){y+=r}if(i){y+=i}var b=N("span",[c],y,o);if(a){for(var w in a){if(a.hasOwnProperty(w)&&w!="style"&&w!="class"){b.setAttribute(w,a[w])}}}return e.content.appendChild(b)}e.content.appendChild(c)}function gn(e,t){if(e.length>1&&!/ /.test(e)){return e}var n=t,r="";for(var i=0;is&&u.from<=s){break}}if(u.to>=l){return f(e,t,n,r,i,o,a)}f(e,t.slice(0,u.to-s),n,r,null,o,a);r=null;t=t.slice(u.to-s);s=u.to}}}function bn(e,t,n,r){var i=!r&&n.widgetNode;if(i){e.map.push(e.pos,e.pos+t,i)}if(!r&&e.cm.display.input.needsContentAttribute){if(!i){i=e.content.appendChild(document.createElement("span"))}i.setAttribute("cm-marker",n.id)}if(i){e.cm.display.input.setUneditable(i);e.content.appendChild(i)}e.pos+=t;e.trailingSpace=false}function wn(e,t,n){var r=e.markedSpans,i=e.text,o=0;if(!r){for(var a=1;al||S.collapsed&&x.to==l&&x.from==l)){if(x.to!=null&&x.to!=l&&p>x.to){p=x.to;v=""}if(S.className){h+=" "+S.className}if(S.css){d=(d?d+";":"")+S.css}if(S.startStyle&&x.from==l){m+=" "+S.startStyle}if(S.endStyle&&x.to==p){(w||(w=[])).push(S.endStyle,x.to)}if(S.title){(y||(y={})).title=S.title}if(S.attributes){for(var T in S.attributes){(y||(y={}))[T]=S.attributes[T]}}if(S.collapsed&&(!g||Gt(g.marker,S)<0)){g=x}}else if(x.from>l&&p>x.from){p=x.from}}if(w){for(var O=0;O=s){break}var k=Math.min(s,p);while(true){if(c){var _=l+c.length;if(!g){var A=_>k?c.slice(0,k-l):c;t.addToken(t,A,f?f+h:h,m,l+A.length==p?v:"",d,y)}if(_>=k){c=c.slice(k-l);l=k;break}l=_;m=""}c=i.slice(o,o=n[u++]);f=pn(n[u++],t.cm.options)}}}function En(e,t,n){this.line=t;this.rest=Zt(t);this.size=this.rest?et(K(this.rest))-n+1:1;this.node=this.text=null;this.hidden=nn(e,t)}function xn(e,t,n){var r=[],i;for(var o=t;o2){o.push((l.bottom+u.top)/2-n.top)}}}o.push(n.bottom-n.top)}}function Qn(e,t,n){if(e.line==t){return{map:e.measure.map,cache:e.measure.cache}}for(var r=0;rn){return{map:e.measure.maps[i],cache:e.measure.caches[i],before:true}}}}function Jn(e,t){t=Qt(t);var n=et(t);var r=e.display.externalMeasured=new En(e.doc,t,n);r.lineN=n;var i=r.built=hn(e,r);r.text=i.pre;_(e.display.lineMeasure,i.pre);return r}function Zn(e,t,n,r){return nr(e,tr(e,t),n,r)}function er(e,t){if(t>=e.display.viewFrom&&t=n.lineN&&tt){o=l-s;i=o-1;if(t>=l){a="right"}}if(i!=null){r=e[u+2];if(s==l&&n==(r.insertLeft?"left":"right")){a=n}if(n=="left"&&i==0){while(u&&e[u-2]==e[u-3]&&e[u-1].insertLeft){r=e[(u-=3)+2];a="left"}}if(n=="right"&&i==l-s){while(u=0;i--){if((n=e[i]).left!=n.right){break}}}return n}function sr(e,t,n,r){var i=or(t.map,n,r);var o=i.node,a=i.start,s=i.end,l=i.collapse;var u;if(o.nodeType==3){for(var c=0;c<4;c++){while(a&&oe(t.line.text.charAt(i.coverStart+a))){--a}while(i.coverStart+s0){l=r="right"}var f;if(e.options.lineWrapping&&(f=o.getClientRects()).length>1){u=f[r=="right"?f.length-1:0]}else{u=o.getBoundingClientRect()}}if(E&&x<9&&!a&&(!u||!u.left&&!u.right)){var d=o.parentNode.getClientRects()[0];if(d){u={left:d.left,right:d.left+Ar(e.display),top:d.top,bottom:d.bottom}}else{u=rr}}var p=u.top-t.rect.top,h=u.bottom-t.rect.top;var v=(p+h)/2;var m=t.view.measure.heights;var g=0;for(;g=o.text.length){t=o.text.length;n="before"}else if(t<=0){t=0;n="after"}if(!u){return l(n=="before"?t-1:t,n=="before")}function c(e,t,n){var r=u[t],i=r.level==1;return l(n?e-1:e,i!=n)}var f=ce(u,t,n);var d=ue;var p=c(t,f,n=="before");if(d!=null){p.other=c(t,d,n!="before")}return p}function br(e,t){var n=0;t=ft(e.doc,t);if(!e.options.lineWrapping){n=Ar(e.display)*t.ch}var r=$e(e.doc,t.line);var i=on(r)+Vn(e.display);return{left:n,right:n,top:i,bottom:i+r.height}}function wr(e,t,n,r,i){var o=it(e,t,n);o.xRel=i;if(r){o.outside=r}return o}function Er(e,t,n){var r=e.doc;n+=e.display.viewOffset;if(n<0){return wr(r.first,0,null,-1,-1)}var i=tt(r,n),o=r.first+r.size-1;if(i>o){return wr(r.first+r.size-1,$e(r,o).text.length,null,1,1)}if(t<0){t=0}var a=$e(r,i);for(;;){var s=Or(e,a,i,t,n);var l=Kt(a,s.ch+(s.xRel>0||s.outside>0?1:0));if(!l){return s}var u=l.find(1);if(u.line==i){return u}a=$e(r,i=u.line)}}function xr(t,e,n,r){r-=hr(e);var i=e.text.length;var o=se(function(e){return nr(t,n,e-1).bottom<=r},i,0);i=se(function(e){return nr(t,n,e).top>r},o,i);return{begin:o,end:i}}function Sr(e,t,n,r){if(!n){n=tr(e,t)}var i=vr(e,t,nr(e,n,r),"line").top;return xr(e,t,n,i)}function Tr(e,t,n,r){return e.bottom<=n?false:e.top>n?true:(r?e.left:e.right)>t}function Or(n,e,t,r,i){i-=on(e);var o=tr(n,e);var a=hr(e);var s=0,l=e.text.length,u=true;var c=de(e,n.doc.direction);if(c){var f=(n.options.lineWrapping?kr:Cr)(n,e,t,o,c,r,i);u=f.level!=1;s=u?f.from:f.to-1;l=u?f.to:f.from-1}var d=null,p=null;var h=se(function(e){var t=nr(n,o,e);t.top+=a;t.bottom+=a;if(!Tr(t,r,i,false)){return false}if(t.top<=i&&t.left<=r){d=e;p=t}return true},s,l);var v,m,g=false;if(p){var y=r-p.left=w.bottom?1:0}h=ae(e.text,h,1);return wr(t,h,m,g,r-v)}function Cr(r,i,o,a,s,l,u){var e=se(function(e){var t=s[e],n=t.level!=1;return Tr(yr(r,it(o,n?t.to:t.from,n?"before":"after"),"line",i,a),l,u,true)},0,s.length-1);var t=s[e];if(e>0){var n=t.level!=1;var c=yr(r,it(o,n?t.from:t.to,n?"after":"before"),"line",i,a);if(Tr(c,l,u,true)&&c.top>u){t=s[e-1]}}return t}function kr(e,t,n,r,i,o,a){var s=xr(e,t,r,a);var l=s.begin;var u=s.end;if(/\s/.test(t.text.charAt(u-1))){u--}var c=null,f=null;for(var d=0;d=u||p.to<=l){continue}var h=p.level!=1;var v=nr(e,r,h?Math.min(u,p.to)-1:Math.max(l,p.from)).right;var m=vm){c=p;f=m}}if(!c){c=i[i.length-1]}if(c.fromu){c={from:c.from,to:u,level:c.level}}return c}function _r(e){if(e.cachedTextHeight!=null){return e.cachedTextHeight}if(ir==null){ir=N("pre",null,"CodeMirror-line-like");for(var t=0;t<49;++t){ir.appendChild(document.createTextNode("x"));ir.appendChild(N("br"))}ir.appendChild(document.createTextNode("x"))}_(e.measure,ir);var n=ir.offsetHeight/50;if(n>3){e.cachedTextHeight=n}k(e.measure);return n||1}function Ar(e){if(e.cachedCharWidth!=null){return e.cachedCharWidth}var t=N("span","xxxxxxxxxx");var n=N("pre",[t],"CodeMirror-line-like");_(e.measure,n);var r=t.getBoundingClientRect(),i=(r.right-r.left)/10;if(i>2){e.cachedCharWidth=i}return i||10}function Ir(e){var t=e.display,n={},r={};var i=t.gutters.clientLeft;for(var o=t.gutters.firstChild,a=0;o;o=o.nextSibling,++a){var s=e.display.gutterSpecs[a].className;n[s]=o.offsetLeft+o.clientLeft+i;r[s]=o.clientWidth}return{fixedPos:Nr(t),gutterTotalWidth:t.gutters.offsetWidth,gutterLeft:n,gutterWidth:r,wrapperWidth:t.wrapper.clientWidth}}function Nr(e){return e.scroller.getBoundingClientRect().left-e.sizer.getBoundingClientRect().left}function Lr(r){var i=_r(r.display),o=r.options.lineWrapping;var a=o&&Math.max(5,r.display.scroller.clientWidth/Ar(r.display)-3);return function(e){if(nn(r.doc,e)){return 0}var t=0;if(e.widgets){for(var n=0;n0&&(u=$e(e.doc,l.line).text).length==l.ch){var c=F(u,u.length,e.options.tabSize)-u.length;l=it(l.line,Math.max(0,Math.round((o-qn(e.display).left)/Ar(e.display))-c))}return l}function Mr(e,t){if(t>=e.display.viewTo){return null}t-=e.display.viewFrom;if(t<0){return null}var n=e.display.view;for(var r=0;rt)){i.updateLineNumbers=t}e.curOp.viewChanged=true;if(t>=i.viewTo){if(At&&en(e.doc,t)i.viewFrom){Fr(e)}else{i.viewFrom+=r;i.viewTo+=r}}else if(t<=i.viewFrom&&n>=i.viewTo){Fr(e)}else if(t<=i.viewFrom){var o=Hr(e,n,n+r,1);if(o){i.view=i.view.slice(o.index);i.viewFrom=o.lineN;i.viewTo+=r}else{Fr(e)}}else if(n>=i.viewTo){var a=Hr(e,t,t,-1);if(a){i.view=i.view.slice(0,a.index);i.viewTo=a.lineN}else{Fr(e)}}else{var s=Hr(e,t,t,-1);var l=Hr(e,n,n+r,1);if(s&&l){i.view=i.view.slice(0,s.index).concat(xn(e,s.lineN,l.lineN)).concat(i.view.slice(l.index));i.viewTo+=r}else{Fr(e)}}var u=i.externalMeasured;if(u){if(n=i.lineN&&t=r.viewTo){return}var o=r.view[Mr(e,t)];if(o.node==null){return}var a=o.changes||(o.changes=[]);if(W(a,n)==-1){a.push(n)}}function Fr(e){e.display.viewFrom=e.display.viewTo=e.doc.first;e.display.view=[];e.display.viewOffset=0}function Hr(e,t,n,r){var i=Mr(e,t),o,a=e.display.view;if(!At||n==e.doc.first+e.doc.size){return{index:i,lineN:n}}var s=e.display.viewFrom;for(var l=0;l0){if(i==a.length-1){return null}o=s+a[i].size-t;i++}else{o=s-t}t+=o;n+=o}while(en(e.doc,n)!=n){if(i==(r<0?0:a.length-1)){return null}n+=r*a[i-(r<0?1:0)].size;i+=r}return{index:i,lineN:n}}function Wr(e,t,n){var r=e.display,i=r.view;if(i.length==0||t>=r.viewTo||n<=r.viewFrom){r.view=xn(e,t,n);r.viewFrom=t}else{if(r.viewFrom>t){r.view=xn(e,t,r.viewFrom).concat(r.view)}else if(r.viewFromn){r.view=r.view.slice(0,Mr(e,n))}}r.viewTo=n}function Ur(e){var t=e.display.view,n=0;for(var r=0;r=e.display.viewTo||s.to().line0){t.blinker=setInterval(function(){return t.cursorDiv.style.visibility=(n=!n)?"":"hidden"},e.options.cursorBlinkRate)}else if(e.options.cursorBlinkRate<0){t.cursorDiv.style.visibility="hidden"}}function Xr(e){if(!e.state.focused){e.display.input.focus();$r(e)}}function Kr(e){e.state.delayingBlurEvent=true;setTimeout(function(){if(e.state.delayingBlurEvent){e.state.delayingBlurEvent=false;Qr(e)}},100)}function $r(e,t){if(e.state.delayingBlurEvent){e.state.delayingBlurEvent=false}if(e.options.readOnly=="nocursor"){return}if(!e.state.focused){ge(e,"focus",e,t);e.state.focused=true;R(e.display.wrapper,"CodeMirror-focused");if(!e.curOp&&e.display.selForContextMenu!=e.doc.sel){e.display.input.reset();if(m){setTimeout(function(){return e.display.input.reset(true)},20)}}e.display.input.receivedFocus()}Yr(e)}function Qr(e,t){if(e.state.delayingBlurEvent){return}if(e.state.focused){ge(e,"blur",e,t);e.state.focused=false;O(e.display.wrapper,"CodeMirror-focused")}clearInterval(e.display.blinker);setTimeout(function(){if(!e.state.focused){e.display.shift=false}},150)}function Jr(e){var t=e.display;var n=t.lineDiv.offsetTop;for(var r=0;r.005||c<-.005){Ze(i.line,a);Zr(i.line);if(i.rest){for(var f=0;fe.display.sizerWidth){var d=Math.ceil(s/Ar(e.display));if(d>e.display.maxLineLength){e.display.maxLineLength=d;e.display.maxLine=i.line;e.display.maxLineChanged=true}}}}function Zr(e){if(e.widgets){for(var t=0;t=a){o=tt(t,on($e(t,l))-e.wrapper.clientHeight);a=l}}return{from:o,to:Math.max(a,o+1)}}function ti(e,t){if(ye(e,"scrollCursorIntoView")){return}var n=e.display,r=n.sizer.getBoundingClientRect(),i=null;if(t.top+r.top<0){i=true}else if(t.bottom+r.top>(window.innerHeight||document.documentElement.clientHeight)){i=false}if(i!=null&&!u){var o=N("div","​",null,"position: absolute;\n top: "+(t.top-n.viewOffset-Vn(e.display))+"px;\n height: "+(t.bottom-t.top+Yn(e)+n.barHeight)+"px;\n left: "+t.left+"px; width: "+Math.max(2,t.right-t.left)+"px;");e.display.lineSpace.appendChild(o);o.scrollIntoView(i);e.display.lineSpace.removeChild(o)}}function ni(e,t,n,r){if(r==null){r=0}var i;if(!e.options.lineWrapping&&t==n){t=t.ch?it(t.line,t.sticky=="before"?t.ch-1:t.ch,"after"):t;n=t.sticky=="before"?it(t.line,t.ch+1,"before"):t}for(var o=0;o<5;o++){var a=false;var s=yr(e,t);var l=!n||n==t?s:yr(e,n);i={left:Math.min(s.left,l.left),top:Math.min(s.top,l.top)-r,right:Math.max(s.left,l.left),bottom:Math.max(s.bottom,l.bottom)+r};var u=ii(e,i);var c=e.doc.scrollTop,f=e.doc.scrollLeft;if(u.scrollTop!=null){fi(e,u.scrollTop);if(Math.abs(e.doc.scrollTop-c)>1){a=true}}if(u.scrollLeft!=null){pi(e,u.scrollLeft);if(Math.abs(e.doc.scrollLeft-f)>1){a=true}}if(!a){break}}return i}function ri(e,t){var n=ii(e,t);if(n.scrollTop!=null){fi(e,n.scrollTop)}if(n.scrollLeft!=null){pi(e,n.scrollLeft)}}function ii(e,t){var n=e.display,r=_r(e.display);if(t.top<0){t.top=0}var i=e.curOp&&e.curOp.scrollTop!=null?e.curOp.scrollTop:n.scroller.scrollTop;var o=Kn(e),a={};if(t.bottom-t.top>o){t.bottom=t.top+o}var s=e.doc.height+Gn(n);var l=t.tops-r;if(t.topi+o){var c=Math.min(t.top,(u?s:t.bottom)-o);if(c!=i){a.scrollTop=c}}var f=e.curOp&&e.curOp.scrollLeft!=null?e.curOp.scrollLeft:n.scroller.scrollLeft;var d=Xn(e)-(e.options.fixedGutter?n.gutters.offsetWidth:0);var p=t.right-t.left>d;if(p){t.right=t.left+d}if(t.left<10){a.scrollLeft=0}else if(t.leftd+f-3){a.scrollLeft=t.right+(p?0:10)-d}return a}function oi(e,t){if(t==null){return}ui(e);e.curOp.scrollTop=(e.curOp.scrollTop==null?e.doc.scrollTop:e.curOp.scrollTop)+t}function ai(e){ui(e);var t=e.getCursor();e.curOp.scrollToPos={from:t,to:t,margin:e.options.cursorScrollMargin}}function si(e,t,n){if(t!=null||n!=null){ui(e)}if(t!=null){e.curOp.scrollLeft=t}if(n!=null){e.curOp.scrollTop=n}}function li(e,t){ui(e);e.curOp.scrollToPos=t}function ui(e){var t=e.curOp.scrollToPos;if(t){e.curOp.scrollToPos=null;var n=br(e,t.from),r=br(e,t.to);ci(e,n,r,t.margin)}}function ci(e,t,n,r){var i=ii(e,{left:Math.min(t.left,n.left),top:Math.min(t.top,n.top)-r,right:Math.max(t.right,n.right),bottom:Math.max(t.bottom,n.bottom)+r});si(e,i.scrollLeft,i.scrollTop)}function fi(e,t){if(Math.abs(e.doc.scrollTop-t)<2){return}if(!v){zi(e,{top:t})}di(e,t,true);if(v){zi(e)}Pi(e,100)}function di(e,t,n){t=Math.max(0,Math.min(e.display.scroller.scrollHeight-e.display.scroller.clientHeight,t));if(e.display.scroller.scrollTop==t&&!n){return}e.doc.scrollTop=t;e.display.scrollbars.setScrollTop(t);if(e.display.scroller.scrollTop!=t){e.display.scroller.scrollTop=t}}function pi(e,t,n,r){t=Math.max(0,Math.min(t,e.display.scroller.scrollWidth-e.display.scroller.clientWidth));if((n?t==e.doc.scrollLeft:Math.abs(e.doc.scrollLeft-t)<2)&&!r){return}e.doc.scrollLeft=t;qi(e);if(e.display.scroller.scrollLeft!=t){e.display.scroller.scrollLeft=t}e.display.scrollbars.setScrollLeft(t)}function hi(e){var t=e.display,n=t.gutters.offsetWidth;var r=Math.round(e.doc.height+Gn(e.display));return{clientHeight:t.scroller.clientHeight,viewHeight:t.wrapper.clientHeight,scrollWidth:t.scroller.scrollWidth,clientWidth:t.scroller.clientWidth,viewWidth:t.wrapper.clientWidth,barLeft:e.options.fixedGutter?n:0,docHeight:r,scrollHeight:r+Yn(e)+t.barHeight,nativeBarWidth:t.nativeBarWidth,gutterWidth:n}}var vi=function(e,t,n){this.cm=n;var r=this.vert=N("div",[N("div",null,null,"min-width: 1px")],"CodeMirror-vscrollbar");var i=this.horiz=N("div",[N("div",null,null,"height: 100%; min-height: 1px")],"CodeMirror-hscrollbar");r.tabIndex=i.tabIndex=-1;e(r);e(i);he(r,"scroll",function(){if(r.clientHeight){t(r.scrollTop,"vertical")}});he(i,"scroll",function(){if(i.clientWidth){t(i.scrollLeft,"horizontal")}});this.checkedZeroWidth=false;if(E&&x<8){this.horiz.style.minHeight=this.vert.style.minWidth="18px"}};vi.prototype.update=function(e){var t=e.scrollWidth>e.clientWidth+1;var n=e.scrollHeight>e.clientHeight+1;var r=e.nativeBarWidth;if(n){this.vert.style.display="block";this.vert.style.bottom=t?r+"px":"0";var i=e.viewHeight-(t?r:0);this.vert.firstChild.style.height=Math.max(0,e.scrollHeight-e.clientHeight+i)+"px"}else{this.vert.style.display="";this.vert.firstChild.style.height="0"}if(t){this.horiz.style.display="block";this.horiz.style.right=n?r+"px":"0";this.horiz.style.left=e.barLeft+"px";var o=e.viewWidth-e.barLeft-(n?r:0);this.horiz.firstChild.style.width=Math.max(0,e.scrollWidth-e.clientWidth+o)+"px"}else{this.horiz.style.display="";this.horiz.firstChild.style.width="0"}if(!this.checkedZeroWidth&&e.clientHeight>0){if(r==0){this.zeroWidthHack()}this.checkedZeroWidth=true}return{right:n?r:0,bottom:t?r:0}},vi.prototype.setScrollLeft=function(e){if(this.horiz.scrollLeft!=e){this.horiz.scrollLeft=e}if(this.disableHoriz){this.enableZeroWidthBar(this.horiz,this.disableHoriz,"horiz")}},vi.prototype.setScrollTop=function(e){if(this.vert.scrollTop!=e){this.vert.scrollTop=e}if(this.disableVert){this.enableZeroWidthBar(this.vert,this.disableVert,"vert")}},vi.prototype.zeroWidthHack=function(){var e=y&&!l?"12px":"18px";this.horiz.style.height=this.vert.style.width=e;this.horiz.style.pointerEvents=this.vert.style.pointerEvents="none";this.disableHoriz=new H;this.disableVert=new H},vi.prototype.enableZeroWidthBar=function(n,r,i){n.style.pointerEvents="auto";function o(){var e=n.getBoundingClientRect();var t=i=="vert"?document.elementFromPoint(e.right-1,(e.top+e.bottom)/2):document.elementFromPoint((e.right+e.left)/2,e.bottom-1);if(t!=n){n.style.pointerEvents="none"}else{r.set(1e3,o)}}r.set(1e3,o)},vi.prototype.clear=function(){var e=this.horiz.parentNode;e.removeChild(this.horiz);e.removeChild(this.vert)};var mi=function(){};function gi(e,t){if(!t){t=hi(e)}var n=e.display.barWidth,r=e.display.barHeight;yi(e,t);for(var i=0;i<4&&n!=e.display.barWidth||r!=e.display.barHeight;i++){if(n!=e.display.barWidth&&e.options.lineWrapping){Jr(e)}yi(e,hi(e));n=e.display.barWidth;r=e.display.barHeight}}function yi(e,t){var n=e.display;var r=n.scrollbars.update(t);n.sizer.style.paddingRight=(n.barWidth=r.right)+"px";n.sizer.style.paddingBottom=(n.barHeight=r.bottom)+"px";n.heightForcer.style.borderBottom=r.bottom+"px solid transparent";if(r.right&&r.bottom){n.scrollbarFiller.style.display="block";n.scrollbarFiller.style.height=r.bottom+"px";n.scrollbarFiller.style.width=r.right+"px"}else{n.scrollbarFiller.style.display=""}if(r.bottom&&e.options.coverGutterNextToScrollbar&&e.options.fixedGutter){n.gutterFiller.style.display="block";n.gutterFiller.style.height=r.bottom+"px";n.gutterFiller.style.width=t.gutterWidth+"px"}else{n.gutterFiller.style.display=""}}mi.prototype.update=function(){return{bottom:0,right:0}},mi.prototype.setScrollLeft=function(){},mi.prototype.setScrollTop=function(){},mi.prototype.clear=function(){};var bi={native:vi,null:mi};function wi(n){if(n.display.scrollbars){n.display.scrollbars.clear();if(n.display.scrollbars.addClass){O(n.display.wrapper,n.display.scrollbars.addClass)}}n.display.scrollbars=new bi[n.options.scrollbarStyle](function(e){n.display.wrapper.insertBefore(e,n.display.scrollbarFiller);he(e,"mousedown",function(){if(n.state.focused){setTimeout(function(){return n.display.input.focus()},0)}});e.setAttribute("cm-not-content","true")},function(e,t){if(t=="horizontal"){pi(n,e)}else{fi(n,e)}},n);if(n.display.scrollbars.addClass){R(n.display.wrapper,n.display.scrollbars.addClass)}}var Ei=0;function xi(e){e.curOp={cm:e,viewChanged:false,startHeight:e.doc.height,forceUpdate:false,updateInput:0,typing:false,changeObjs:null,cursorActivityHandlers:null,cursorActivityCalled:0,selectionChanged:false,updateMaxLine:false,scrollLeft:null,scrollTop:null,scrollToPos:null,focus:false,id:++Ei};Tn(e.curOp)}function Si(e){var t=e.curOp;if(t){Cn(t,function(e){for(var t=0;t=n.viewTo)||n.maxLineChanged&&t.options.lineWrapping;e.update=e.mustUpdate&&new Di(t,e.mustUpdate&&{top:e.scrollTop,ensure:e.scrollToPos},e.forceUpdate)}function Ci(e){e.updatedDisplay=e.mustUpdate&&Wi(e.cm,e.update)}function ki(e){var t=e.cm,n=t.display;if(e.updatedDisplay){Jr(t)}e.barMeasure=hi(t);if(n.maxLineChanged&&!t.options.lineWrapping){e.adjustWidthTo=Zn(t,n.maxLine,n.maxLine.text.length).left+3;t.display.sizerWidth=e.adjustWidthTo;e.barMeasure.scrollWidth=Math.max(n.scroller.clientWidth,n.sizer.offsetLeft+e.adjustWidthTo+Yn(t)+t.display.barWidth);e.maxScrollLeft=Math.max(0,n.sizer.offsetLeft+e.adjustWidthTo-Xn(t))}if(e.updatedDisplay||e.selectionChanged){e.preparedSelection=n.input.prepareSelection()}}function _i(e){var t=e.cm;if(e.adjustWidthTo!=null){t.display.sizer.style.minWidth=e.adjustWidthTo+"px";if(e.maxScrollLeft=l.display.viewTo){return}var c=+new Date+l.options.workTime;var f=yt(l,u.highlightFrontier);var d=[];u.iter(f.line,Math.min(u.first+u.size,l.display.viewTo+500),function(e){if(f.line>=l.display.viewFrom){var t=e.styles;var n=e.text.length>l.options.maxHighlightLength?qe(u.mode,f.state):null;var r=mt(l,e,f,true);if(n){f.state=n}e.styles=r.styles;var i=e.styleClasses,o=r.classes;if(o){e.styleClasses=o}else if(i){e.styleClasses=null}var a=!t||t.length!=e.styles.length||i!=o&&(!i||!o||i.bgClass!=o.bgClass||i.textClass!=o.textClass);for(var s=0;!a&&sc){Pi(l,l.options.workDelay);return true}});u.highlightFrontier=f.line;u.modeFrontier=Math.max(u.modeFrontier,f.line);if(d.length){Ii(l,function(){for(var e=0;e=n.viewFrom&&t.visible.to<=n.viewTo&&(n.updateLineNumbers==null||n.updateLineNumbers>=n.viewTo)&&n.renderedView==n.view&&Ur(e)==0){return false}if(Yi(e)){Fr(e);t.dims=Ir(e)}var i=r.first+r.size;var o=Math.max(t.visible.from-e.options.viewportMargin,r.first);var a=Math.min(i,t.visible.to+e.options.viewportMargin);if(n.viewFroma&&n.viewTo-a<20){a=Math.min(i,n.viewTo)}if(At){o=en(e.doc,o);a=tn(e.doc,a)}var s=o!=n.viewFrom||a!=n.viewTo||n.lastWrapHeight!=t.wrapperHeight||n.lastWrapWidth!=t.wrapperWidth;Wr(e,o,a);n.viewOffset=on($e(e.doc,n.viewFrom));e.display.mover.style.top=n.viewOffset+"px";var l=Ur(e);if(!s&&l==0&&!t.force&&n.renderedView==n.view&&(n.updateLineNumbers==null||n.updateLineNumbers>=n.viewTo)){return false}var u=Fi(e);if(l>4){n.lineDiv.style.display="none"}Bi(e,n.updateLineNumbers,t.dims);if(l>4){n.lineDiv.style.display=""}n.renderedView=n.view;Hi(u);k(n.cursorDiv);k(n.selectionDiv);n.gutters.style.height=n.sizer.style.minHeight=0;if(s){n.lastWrapHeight=t.wrapperHeight;n.lastWrapWidth=t.wrapperWidth;Pi(e,400)}n.updateLineNumbers=null;return true}function Ui(e,t){var n=t.viewport;for(var r=true;;r=false){if(!r||!e.options.lineWrapping||t.oldDisplayWidth==Xn(e)){if(n&&n.top!=null){n={top:Math.min(e.doc.height+Gn(e.display)-Kn(e),n.top)}}t.visible=ei(e.display,e.doc,n);if(t.visible.from>=e.display.viewFrom&&t.visible.to<=e.display.viewTo){break}}else if(r){t.visible=ei(e.display,e.doc,n)}if(!Wi(e,t)){break}Jr(e);var i=hi(e);zr(e);gi(e,i);Gi(e,i);t.force=false}t.signal(e,"update",e);if(e.display.viewFrom!=e.display.reportedViewFrom||e.display.viewTo!=e.display.reportedViewTo){t.signal(e,"viewportChange",e,e.display.viewFrom,e.display.viewTo);e.display.reportedViewFrom=e.display.viewFrom;e.display.reportedViewTo=e.display.viewTo}}function zi(e,t){var n=new Di(e,t);if(Wi(e,n)){Jr(e);Ui(e,n);var r=hi(e);zr(e);gi(e,r);Gi(e,r);n.finish()}}function Bi(n,e,t){var r=n.display,i=n.options.lineNumbers;var o=r.lineDiv,a=o.firstChild;function s(e){var t=e.nextSibling;if(m&&y&&n.display.currentWheelTarget==e){e.style.display="none"}else{e.parentNode.removeChild(e)}return t}var l=r.view,u=r.viewFrom;for(var c=0;c-1){p=false}In(n,f,u,t)}if(p){k(f.lineNumber);f.lineNumber.appendChild(document.createTextNode(rt(n.options,u)))}a=f.node.nextSibling}u+=f.size}while(a){a=s(a)}}function Vi(e){var t=e.gutters.offsetWidth;e.sizer.style.marginLeft=t+"px"}function Gi(e,t){e.display.sizer.style.minHeight=t.docHeight+"px";e.display.heightForcer.style.top=t.docHeight+"px";e.display.gutters.style.height=t.docHeight+e.display.barHeight+Yn(e)+"px"}function qi(e){var t=e.display,n=t.view;if(!t.alignWidgets&&(!t.gutters.firstChild||!e.options.fixedGutter)){return}var r=Nr(t)-t.scroller.scrollLeft+e.doc.scrollLeft;var i=t.gutters.offsetWidth,o=r+"px";for(var a=0;aa.clientWidth;var l=a.scrollHeight>a.clientHeight;if(!(r&&s||i&&l)){return}if(i&&y&&m){e:for(var u=t.target,c=o.view;u!=a;u=u.parentNode){for(var f=0;f=0&&ot(e,r.to())<=0){return n}}return-1};var io=function(e,t){this.anchor=e;this.head=t};function oo(e,t,n){var r=e&&e.options.selectionsMayTouch;var i=t[n];t.sort(function(e,t){return ot(e.from(),t.from())});n=W(t,i);for(var o=1;o0:l>=0){var u=ut(s.from(),a.from()),c=lt(s.to(),a.to());var f=s.empty()?a.from()==a.head:s.from()==s.head;if(o<=n){--n}t.splice(--o,2,new io(f?c:u,f?u:c))}}return new ro(t,n)}function ao(e,t){return new ro([new io(e,t||e)],0)}function so(e){if(!e.text){return e.to}return it(e.from.line+e.text.length-1,K(e.text).length+(e.text.length==1?e.from.ch:0))}function lo(e,t){if(ot(e,t.from)<0){return e}if(ot(e,t.to)<=0){return so(t)}var n=e.line+t.text.length-(t.to.line-t.from.line)-1,r=e.ch;if(e.line==t.to.line){r+=so(t).ch-t.to.ch}return it(n,r)}function uo(e,t){var n=[];for(var r=0;r1){e.remove(s.line+1,h-1)}e.insert(s.line+1,g)}_n(e,"change",e,r)}function go(e,a,s){function l(e,t,n){if(e.linked){for(var r=0;r1&&!e.done[e.done.length-2].ranges){e.done.pop();return K(e.done)}}function Oo(e,t,n,r){var i=e.history;i.undone.length=0;var o=+new Date,a;var s;if((i.lastOp==r||i.lastOrigin==t.origin&&t.origin&&(t.origin.charAt(0)=="+"&&i.lastModTime>o-(e.cm?e.cm.options.historyEventDelay:500)||t.origin.charAt(0)=="*"))&&(a=To(i,i.lastOp==r))){s=K(a.changes);if(ot(t.from,t.to)==0&&ot(t.from,s.to)==0){s.to=so(t)}else{a.changes.push(xo(e,t))}}else{var l=K(i.done);if(!l||!l.ranges){_o(e.sel,i.done)}a={changes:[xo(e,t)],generation:i.generation};i.done.push(a);while(i.done.length>i.undoDepth){i.done.shift();if(!i.done[0].ranges){i.done.shift()}}}i.done.push(n);i.generation=++i.maxGeneration;i.lastModTime=i.lastSelTime=o;i.lastOp=i.lastSelOp=r;i.lastOrigin=i.lastSelOrigin=t.origin;if(!s){ge(e,"historyAdded")}}function Co(e,t,n,r){var i=t.charAt(0);return i=="*"||i=="+"&&n.ranges.length==r.ranges.length&&n.somethingSelected()==r.somethingSelected()&&new Date-e.history.lastSelTime<=(e.cm?e.cm.options.historyEventDelay:500)}function ko(e,t,n,r){var i=e.history,o=r&&r.origin;if(n==i.lastSelOp||o&&i.lastSelOrigin==o&&(i.lastModTime==i.lastSelTime&&i.lastOrigin==o||Co(e,o,K(i.done),t))){i.done[i.done.length-1]=t}else{_o(t,i.done)}i.lastSelTime=+new Date;i.lastSelOrigin=o;i.lastSelOp=n;if(r&&r.clearRedo!==false){So(i.undone)}}function _o(e,t){var n=K(t);if(!(n&&n.ranges&&n.equals(e))){t.push(e)}}function Ao(t,n,e,r){var i=n["spans_"+t.id],o=0;t.iter(Math.max(t.first,e),Math.min(t.first+t.size,r),function(e){if(e.markedSpans){(i||(i=n["spans_"+t.id]={}))[o]=e.markedSpans}++o})}function Io(e){if(!e){return null}var t;for(var n=0;n-1){K(s)[f]=u[f];delete u[f]}}}}}}return r}function Po(e,t,n,r){if(r){var i=e.anchor;if(n){var o=ot(t,i)<0;if(o!=ot(n,i)<0){i=t;t=n}else if(o!=ot(t,n)<0){t=n}}return new io(i,t)}else{return new io(n||t,t)}}function Mo(e,t,n,r,i){if(i==null){i=e.cm&&(e.cm.display.shift||e.extend)}Uo(e,new ro([Po(e.sel.primary(),t,n,i)],0),r)}function Do(e,t,n){var r=[];var i=e.cm&&(e.cm.display.shift||e.extend);for(var o=0;o=t.ch:s.to>t.ch))){if(i){ge(l,"beforeCursorEnter");if(l.explicitlyCleared){if(!o.markedSpans){break}else{--a;continue}}}if(!l.atomic){continue}if(n){var f=l.find(r<0?1:-1),d=void 0;if(r<0?c:u){f=Xo(e,f,-r,f&&f.line==t.line?o:null)}if(f&&f.line==t.line&&(d=ot(f,n))&&(r<0?d<0:d>0)){return qo(e,f,t,r,i)}}var p=l.find(r<0?-1:1);if(r<0?u:c){p=Xo(e,p,r,p.line==t.line?o:null)}return p?qo(e,p,t,r,i):null}}}return t}function Yo(e,t,n,r,i){var o=r||1;var a=qo(e,t,n,o,i)||!i&&qo(e,t,n,o,true)||qo(e,t,n,-o,i)||!i&&qo(e,t,n,-o,true);if(!a){e.cantEdit=true;return it(e.first,0)}return a}function Xo(e,t,n,r){if(n<0&&t.ch==0){if(t.line>e.first){return ft(e,it(t.line-1))}else{return null}}else if(n>0&&t.ch==(r||$e(e,t.line)).text.length){if(t.line=0;--i){Jo(e,{from:r[i].from,to:r[i].to,text:i?[""]:t.text,origin:t.origin})}}else{Jo(e,t)}}function Jo(e,n){if(n.text.length==1&&n.text[0]==""&&ot(n.from,n.to)==0){return}var t=uo(e,n);Oo(e,n,t,e.cm?e.cm.curOp.id:NaN);ta(e,n,t,Ft(e,n));var r=[];go(e,function(e,t){if(!t&&W(r,e.history)==-1){aa(e.history,n);r.push(e.history)}ta(e,n,null,Ft(e,n))})}function Zo(i,o,e){var t=i.cm&&i.cm.state.suppressEdits;if(t&&!e){return}var n=i.history,a,r=i.sel;var s=o=="undo"?n.done:n.undone,l=o=="undo"?n.undone:n.done;var u=0;for(;u=0;--p){var h=d(p);if(h)return h.v}}function ea(e,t){if(t==0){return}e.first+=t;e.sel=new ro($(e.sel.ranges,function(e){return new io(it(e.anchor.line+t,e.anchor.ch),it(e.head.line+t,e.head.ch))}),e.sel.primIndex);if(e.cm){Dr(e.cm,e.first,e.first-t,t);for(var n=e.cm.display,r=n.viewFrom;re.lastLine()){return}if(t.from.lineo){t={from:t.from,to:it(o,$e(e,o).text.length),text:[t.text[0]],origin:t.origin}}t.removed=Qe(e,t.from,t.to);if(!n){n=uo(e,t)}if(e.cm){na(e.cm,t,r)}else{mo(e,t,r)}zo(e,n,B);if(e.cantEdit&&Yo(e,it(e.firstLine(),0))){e.cantEdit=false}}function na(e,t,n){var r=e.doc,i=e.display,o=t.from,a=t.to;var s=false,l=o.line;if(!e.options.lineWrapping){l=et(Qt($e(r,o.line)));r.iter(l,a.line+1,function(e){if(e==i.maxLine){s=true;return true}})}if(r.sel.contains(t.from,t.to)>-1){be(e)}mo(r,t,n,Lr(e));if(!e.options.lineWrapping){r.iter(l,o.line+t.text.length,function(e){var t=an(e);if(t>i.maxLineLength){i.maxLine=e;i.maxLineLength=t;i.maxLineChanged=true;s=false}});if(s){e.curOp.updateMaxLine=true}}kt(r,o.line);Pi(e,400);var u=t.text.length-(a.line-o.line)-1;if(t.full){Dr(e)}else if(o.line==a.line&&t.text.length==1&&!vo(e.doc,t)){jr(e,o.line,"text")}else{Dr(e,o.line,a.line+1,u)}var c=we(e,"changes"),f=we(e,"change");if(f||c){var d={from:o,to:a,text:t.text,removed:t.removed,origin:t.origin};if(f){_n(e,"change",e,d)}if(c){(e.curOp.changeObjs||(e.curOp.changeObjs=[])).push(d)}}e.display.selForContextMenu=null}function ra(e,t,n,r,i){var o;if(!r){r=n}if(ot(r,n)<0){o=[r,n],n=o[0],r=o[1]}if(typeof t=="string"){t=e.splitLines(t)}Qo(e,{from:n,to:r,text:t,origin:i})}function ia(e,t,n,r){if(n1||!(this.children[0]instanceof la))){var s=[];this.collapse(s);this.children=[new la(s)];this.children[0].parent=this}},collapse:function(e){for(var t=0;t50){var a=i.lines.length%25+25;for(var s=a;s10);e.parent.maybeSpill()},iterN:function(e,t,n){for(var r=0;r0||a==0&&o.clearWhenEmpty!==false){return o}if(o.replacedWith){o.collapsed=true;o.widgetNode=A("span",[o.replacedWith],"CodeMirror-widget");if(!e.handleMouseEvents){o.widgetNode.setAttribute("cm-ignore-events","true")}if(e.insertLeft){o.widgetNode.insertLeft=true}}if(o.collapsed){if($t(t,n.line,n,r,o)||n.line!=r.line&&$t(t,r.line,n,r,o)){throw new Error("Inserting collapsed marker partially overlapping an existing one")}Nt()}if(o.addToHistory){Oo(t,{from:n,to:r,origin:"markText"},t.sel,NaN)}var s=n.line,l=t.cm,u;t.iter(s,r.line+1,function(e){if(l&&o.collapsed&&!l.options.lineWrapping&&Qt(e)==l.display.maxLine){u=true}if(o.collapsed&&s!=n.line){Ze(e,0)}Mt(e,new Lt(o,s==n.line?n.ch:null,s==r.line?r.ch:null));++s});if(o.collapsed){t.iter(n.line,r.line+1,function(e){if(nn(t,e)){Ze(e,0)}})}if(o.clearOnEnter){he(o,"beforeCursorEnter",function(){return o.clear()})}if(o.readOnly){It();if(t.history.done.length||t.history.undone.length){t.clearHistory()}}if(o.collapsed){o.id=++pa;o.atomic=true}if(l){if(u){l.curOp.updateMaxLine=true}if(o.collapsed){Dr(l,n.line,r.line+1)}else if(o.className||o.startStyle||o.endStyle||o.css||o.attributes||o.title){for(var c=n.line;c<=r.line;c++){jr(l,c,"text")}}if(o.atomic){Vo(l.doc)}_n(l,"markerAdded",l,o)}return o}ha.prototype.clear=function(){if(this.explicitlyCleared){return}var e=this.doc.cm,t=e&&!e.curOp;if(t){xi(e)}if(we(this,"clear")){var n=this.find();if(n){_n(this,"clear",n.from,n.to)}}var r=null,i=null;for(var o=0;oe.display.maxLineLength){e.display.maxLine=u;e.display.maxLineLength=c;e.display.maxLineChanged=true}}}if(r!=null&&e&&this.collapsed){Dr(e,r,i+1)}this.lines.length=0;this.explicitlyCleared=true;if(this.atomic&&this.doc.cantEdit){this.doc.cantEdit=false;if(e){Vo(e.doc)}}if(e){_n(e,"markerCleared",e,this,r,i)}if(t){Si(e)}if(this.parent){this.parent.clear()}},ha.prototype.find=function(e,t){if(e==null&&this.type=="bookmark"){e=1}var n,r;for(var i=0;i=0;l--){Qo(this,r[l])}if(s){Wo(this,s)}else if(this.cm){ai(this.cm)}}),undo:Ri(function(){Zo(this,"undo")}),redo:Ri(function(){Zo(this,"redo")}),undoSelection:Ri(function(){Zo(this,"undo",true)}),redoSelection:Ri(function(){Zo(this,"redo",true)}),setExtending:function(e){this.extend=e},getExtending:function(){return this.extend},historySize:function(){var e=this.history,t=0,n=0;for(var r=0;r=e.ch)){t.push(i.marker.parent||i.marker)}}}return t},findMarks:function(i,o,a){i=ft(this,i);o=ft(this,o);var s=[],l=i.line;this.iter(i.line,o.line+1,function(e){var t=e.markedSpans;if(t){for(var n=0;n=r.to||r.from==null&&l!=i.line||r.from!=null&&l==o.line&&r.from>=o.ch)&&(!a||a(r.marker))){s.push(r.marker.parent||r.marker)}}}++l});return s},getAllMarks:function(){var r=[];this.iter(function(e){var t=e.markedSpans;if(t){for(var n=0;nn){r=n;return true}n-=t;++i});return ft(this,it(i,r))},indexFromPos:function(e){e=ft(this,e);var t=e.ch;if(e.linet){t=e.from}if(e.to!=null&&e.to-1){r.state.draggingText(e);setTimeout(function(){return r.display.input.focus()},20);return}try{var c=e.dataTransfer.getData("Text");if(c){var f;if(r.state.draggingText&&!r.state.draggingText.copy){f=r.listSelections()}zo(r.doc,ao(t,t));if(f){for(var d=0;d=0;e--){ra(t.doc,"",r[e].from,r[e].to,"+delete")}ai(t)})}function qa(e,t,n){var r=ae(e.text,t+n,n);return r<0||r>e.text.length?null:r}function Ya(e,t,n){var r=qa(e,t.ch,n);return r==null?null:new it(t.line,r,n<0?"after":"before")}function Xa(e,t,n,r,i){if(e){if(t.doc.direction=="rtl"){i=-i}var o=de(n,t.doc.direction);if(o){var a=i<0?K(o):o[0];var s=i<0==(a.level==1);var l=s?"after":"before";var u;if(a.level>0||t.doc.direction=="rtl"){var c=tr(t,n);u=i<0?n.text.length-1:0;var f=nr(t,c,u).top;u=se(function(e){return nr(t,c,e).top==f},i<0==(a.level==1)?a.from:a.to-1,u);if(l=="before"){u=qa(n,u,1)}}else{u=i<0?a.to:a.from}return new it(r,u,l)}}return new it(r,i<0?n.text.length:0,i<0?"before":"after")}function Ka(t,n,s,e){var l=de(n,t.doc.direction);if(!l){return Ya(n,s,e)}if(s.ch>=n.text.length){s.ch=n.text.length;s.sticky="before"}else if(s.ch<=0){s.ch=0;s.sticky="after"}var r=ce(l,s.ch,s.sticky),i=l[r];if(t.doc.direction=="ltr"&&i.level%2==0&&(e>0?i.to>s.ch:i.from=i.from&&d>=c.begin:d<=i.to&&d<=c.end)){var p=f?"before":"after";return new it(s.line,d,p)}}var h=function(e,t,n){var r=function(e,t){return t?new it(s.line,u(e,1),"before"):new it(s.line,e,"after")};for(;e>=0&&e0==(i.level!=1);var a=o?n.begin:u(n.end,-1);if(i.from<=a&&a0?c.end:u(c.begin,-1);if(m!=null&&!(e>0&&m==n.text.length)){v=h(e>0?0:l.length-1,e,a(m));if(v){return v}}return null}ja.basic={Left:"goCharLeft",Right:"goCharRight",Up:"goLineUp",Down:"goLineDown",End:"goLineEnd",Home:"goLineStartSmart",PageUp:"goPageUp",PageDown:"goPageDown",Delete:"delCharAfter",Backspace:"delCharBefore","Shift-Backspace":"delCharBefore",Tab:"defaultTab","Shift-Tab":"indentAuto",Enter:"newlineAndIndent",Insert:"toggleOverwrite",Esc:"singleSelection"},ja.pcDefault={"Ctrl-A":"selectAll","Ctrl-D":"deleteLine","Ctrl-Z":"undo","Shift-Ctrl-Z":"redo","Ctrl-Y":"redo","Ctrl-Home":"goDocStart","Ctrl-End":"goDocEnd","Ctrl-Up":"goLineUp","Ctrl-Down":"goLineDown","Ctrl-Left":"goGroupLeft","Ctrl-Right":"goGroupRight","Alt-Left":"goLineStart","Alt-Right":"goLineEnd","Ctrl-Backspace":"delGroupBefore","Ctrl-Delete":"delGroupAfter","Ctrl-S":"save","Ctrl-F":"find","Ctrl-G":"findNext","Shift-Ctrl-G":"findPrev","Shift-Ctrl-F":"replace","Shift-Ctrl-R":"replaceAll","Ctrl-[":"indentLess","Ctrl-]":"indentMore","Ctrl-U":"undoSelection","Shift-Ctrl-U":"redoSelection","Alt-U":"redoSelection",fallthrough:"basic"},ja.emacsy={"Ctrl-F":"goCharRight","Ctrl-B":"goCharLeft","Ctrl-P":"goLineUp","Ctrl-N":"goLineDown","Alt-F":"goWordRight","Alt-B":"goWordLeft","Ctrl-A":"goLineStart","Ctrl-E":"goLineEnd","Ctrl-V":"goPageDown","Shift-Ctrl-V":"goPageUp","Ctrl-D":"delCharAfter","Ctrl-H":"delCharBefore","Alt-D":"delWordAfter","Alt-Backspace":"delWordBefore","Ctrl-K":"killLine","Ctrl-T":"transposeChars","Ctrl-O":"openLine"},ja.macDefault={"Cmd-A":"selectAll","Cmd-D":"deleteLine","Cmd-Z":"undo","Shift-Cmd-Z":"redo","Cmd-Y":"redo","Cmd-Home":"goDocStart","Cmd-Up":"goDocStart","Cmd-End":"goDocEnd","Cmd-Down":"goDocEnd","Alt-Left":"goGroupLeft","Alt-Right":"goGroupRight","Cmd-Left":"goLineLeft","Cmd-Right":"goLineRight","Alt-Backspace":"delGroupBefore","Ctrl-Alt-Backspace":"delGroupAfter","Alt-Delete":"delGroupAfter","Cmd-S":"save","Cmd-F":"find","Cmd-G":"findNext","Shift-Cmd-G":"findPrev","Cmd-Alt-F":"replace","Shift-Cmd-Alt-F":"replaceAll","Cmd-[":"indentLess","Cmd-]":"indentMore","Cmd-Backspace":"delWrappedLineLeft","Cmd-Delete":"delWrappedLineRight","Cmd-U":"undoSelection","Shift-Cmd-U":"redoSelection","Ctrl-Up":"goDocStart","Ctrl-Down":"goDocEnd",fallthrough:["basic","emacsy"]},ja["default"]=y?ja.macDefault:ja.pcDefault;var $a={selectAll:Ko,singleSelection:function(e){return e.setSelection(e.getCursor("anchor"),e.getCursor("head"),B)},killLine:function(n){return Ga(n,function(e){if(e.empty()){var t=$e(n.doc,e.head.line).text.length;if(e.head.ch==t&&e.head.line0){r=new it(r.line,r.ch+1);a.replaceRange(i.charAt(r.ch-1)+i.charAt(r.ch-2),it(r.line,r.ch-2),r,"+transpose")}else if(r.line>a.doc.first){var o=$e(a.doc,r.line-1).text;if(o){r=new it(r.line,1);a.replaceRange(i.charAt(0)+a.doc.lineSeparator()+o.charAt(o.length-1),it(r.line-1,o.length-1),r,"+transpose")}}}t.push(new io(r,r))}a.setSelections(t)})},newlineAndIndent:function(r){return Ii(r,function(){var e=r.listSelections();for(var t=e.length-1;t>=0;t--){r.replaceRange(r.doc.lineSeparator(),e[t].anchor,e[t].head,"+input")}e=r.listSelections();for(var n=0;n-1&&(ot((a=o.ranges[a]).from(),t)<0||t.xRel>0)&&(ot(a.to(),t)>0||t.xRel<0)){Es(e,r,t,i)}else{Ss(e,r,t,i)}}function Es(t,n,r,i){var o=t.display,a=false;var s=Ni(t,function(e){if(m){o.scroller.draggable=false}t.state.draggingText=false;me(o.wrapper.ownerDocument,"mouseup",s);me(o.wrapper.ownerDocument,"mousemove",l);me(o.scroller,"dragstart",u);me(o.scroller,"drop",s);if(!a){xe(e);if(!i.addNew){Mo(t.doc,r,null,null,i.extend)}if(m||E&&x==9){setTimeout(function(){o.wrapper.ownerDocument.body.focus();o.input.focus()},20)}else{o.input.focus()}}});var l=function(e){a=a||Math.abs(n.clientX-e.clientX)+Math.abs(n.clientY-e.clientY)>=10};var u=function(){return a=true};if(m){o.scroller.draggable=true}t.state.draggingText=s;s.copy=!i.moveOnDrag;if(o.scroller.dragDrop){o.scroller.dragDrop()}he(o.wrapper.ownerDocument,"mouseup",s);he(o.wrapper.ownerDocument,"mousemove",l);he(o.scroller,"dragstart",u);he(o.scroller,"drop",s);Kr(t);setTimeout(function(){return o.input.focus()},20)}function xs(e,t,n){if(n=="char"){return new io(t,t)}if(n=="word"){return e.findWordAt(t)}if(n=="line"){return new io(it(t.line,0),ft(e.doc,it(t.line+1,0)))}var r=n(e,t);return new io(r.from,r.to)}function Ss(m,e,g,y){var o=m.display,b=m.doc;xe(e);var w,E,x=b.sel,t=x.ranges;if(y.addNew&&!y.extend){E=b.sel.contains(g);if(E>-1){w=t[E]}else{w=new io(g,g)}}else{w=b.sel.primary();E=b.sel.primIndex}if(y.unit=="rectangle"){if(!y.addNew){w=new io(g,g)}g=Pr(m,e,true,true);E=-1}else{var n=xs(m,g,y.unit);if(y.extend){w=Po(w,n.anchor,n.head,y.extend)}else{w=n}}if(!y.addNew){E=0;Uo(b,new ro([w],0),V);x=b.sel}else if(E==-1){E=t.length;Uo(b,oo(m,t.concat([w]),E),{scroll:false,origin:"*mouse"})}else if(t.length>1&&t[E].empty()&&y.unit=="char"&&!y.extend){Uo(b,oo(m,t.slice(0,E).concat(t.slice(E+1)),0),{scroll:false,origin:"*mouse"});x=b.sel}else{jo(b,E,w,V)}var S=g;function a(e){if(ot(S,e)==0){return}S=e;if(y.unit=="rectangle"){var t=[],n=m.options.tabSize;var r=F($e(b,g.line).text,g.ch,n);var i=F($e(b,e.line).text,e.ch,n);var o=Math.min(r,i),a=Math.max(r,i);for(var s=Math.min(g.line,e.line),l=Math.min(m.lastLine(),Math.max(g.line,e.line));s<=l;s++){var u=$e(b,s).text,c=q(u,o,n);if(o==a){t.push(new io(it(s,c),it(s,c)))}else if(u.length>c){t.push(new io(it(s,c),it(s,q(u,a,n))))}}if(!t.length){t.push(new io(g,g))}Uo(b,oo(m,x.ranges.slice(0,E).concat(t),E),{origin:"*mouse",scroll:false});m.scrollIntoView(e)}else{var f=w;var d=xs(m,e,y.unit);var p=f.anchor,h;if(ot(d.anchor,p)>0){h=d.head;p=ut(f.from(),d.anchor)}else{h=d.anchor;p=lt(f.to(),d.head)}var v=x.ranges.slice(0);v[E]=Ts(m,new io(ft(b,p),h));Uo(b,oo(m,v,E),V)}}var s=o.wrapper.getBoundingClientRect();var l=0;function u(e){var t=++l;var n=Pr(m,e,true,y.unit=="rectangle");if(!n){return}if(ot(n,S)!=0){m.curOp.focus=L();a(n);var r=ei(o,b);if(n.line>=r.to||n.lines.bottom?20:0;if(i){setTimeout(Ni(m,function(){if(l!=t){return}o.scroller.scrollTop+=i;u(e)}),50)}}}function r(e){m.state.selectingText=false;l=Infinity;if(e){xe(e);o.input.focus()}me(o.wrapper.ownerDocument,"mousemove",i);me(o.wrapper.ownerDocument,"mouseup",c);b.history.lastSelOrigin=null}var i=Ni(m,function(e){if(e.buttons===0||!ke(e)){r(e)}else{u(e)}});var c=Ni(m,r);m.state.selectingText=c;he(o.wrapper.ownerDocument,"mousemove",i);he(o.wrapper.ownerDocument,"mouseup",c)}function Ts(e,t){var n=t.anchor;var r=t.head;var i=$e(e.doc,n.line);if(ot(n,r)==0&&n.sticky==r.sticky){return t}var o=de(i);if(!o){return t}var a=ce(o,n.ch,n.sticky),s=o[a];if(s.from!=n.ch&&s.to!=n.ch){return t}var l=a+(s.from==n.ch==(s.level!=1)?0:1);if(l==0||l==o.length){return t}var u;if(r.line!=n.line){u=(r.line-n.line)*(e.doc.direction=="ltr"?1:-1)>0}else{var c=ce(o,r.ch,r.sticky);var f=c-a||(r.ch-n.ch)*(s.level==1?-1:1);if(c==l-1||c==l){u=f<0}else{u=f>0}}var d=o[l+(u?-1:0)];var p=u==(d.level==1);var h=p?d.from:d.to,v=p?"after":"before";return n.ch==h&&n.sticky==v?t:new io(new it(n.line,h,v),r)}function Os(e,t,n,r){var i,o;if(t.touches){i=t.touches[0].clientX;o=t.touches[0].clientY}else{try{i=t.clientX;o=t.clientY}catch(t){return false}}if(i>=Math.floor(e.display.gutters.getBoundingClientRect().right)){return false}if(r){xe(t)}var a=e.display;var s=a.lineDiv.getBoundingClientRect();if(o>s.bottom||!we(e,n)){return Te(t)}o-=s.top-a.viewOffset;for(var l=0;l=i){var c=tt(e.doc,o);var f=e.display.gutterSpecs[l];ge(e,n,e,c,f.className,t);return Te(t)}}}function Cs(e,t){return Os(e,t,"gutterClick",true)}function ks(e,t){if(Bn(e.display,t)||_s(e,t)){return}if(ye(e,t,"contextmenu")){return}if(!S){e.display.input.onContextMenu(t)}}function _s(e,t){if(!we(e,"gutterContextMenu")){return false}return Os(e,t,"gutterContextMenu",false)}function As(e){e.display.wrapper.className=e.display.wrapper.className.replace(/\s*cm-s-\S+/g,"")+e.options.theme.replace(/(^|\s)\s*/g," cm-s-");fr(e)}ps.prototype.compare=function(e,t,n){return this.time+ds>e&&ot(t,this.pos)==0&&n==this.button};var Is={toString:function(){return"CodeMirror.Init"}},Ns={},Ls={};function Rs(i){var o=i.optionHandlers;function e(e,t,r,n){i.defaults[e]=t;if(r){o[e]=n?function(e,t,n){if(n!=Is){r(e,t,n)}}:r}}i.defineOption=e;i.Init=Is;e("value","",function(e,t){return e.setValue(t)},true);e("mode",null,function(e,t){e.doc.modeOption=t;po(e)},true);e("indentUnit",2,po,true);e("indentWithTabs",false);e("smartIndent",true);e("tabSize",4,function(e){ho(e);fr(e);Dr(e)},true);e("lineSeparator",null,function(e,r){e.doc.lineSep=r;if(!r){return}var i=[],o=e.doc.first;e.doc.iter(function(e){for(var t=0;;){var n=e.text.indexOf(r,t);if(n==-1){break}t=n+r.length;i.push(it(o,n))}o++});for(var t=i.length-1;t>=0;t--){ra(e.doc,r,i[t],it(i[t].line,i[t].ch+r.length))}});e("specialChars",/[\u0000-\u001f\u007f-\u009f\u00ad\u061c\u200b-\u200f\u2028\u2029\ufeff\ufff9-\ufffc]/g,function(e,t,n){e.state.specialChars=new RegExp(t.source+(t.test("\t")?"":"|\t"),"g");if(n!=Is){e.refresh()}});e("specialCharPlaceholder",vn,function(e){return e.refresh()},true);e("electricChars",true);e("inputStyle",d?"contenteditable":"textarea",function(){throw new Error("inputStyle can not (yet) be changed in a running editor")},true);e("spellcheck",false,function(e,t){return e.getInputField().spellcheck=t},true);e("autocorrect",false,function(e,t){return e.getInputField().autocorrect=t},true);e("autocapitalize",false,function(e,t){return e.getInputField().autocapitalize=t},true);e("rtlMoveVisually",!h);e("wholeLineUpdateBefore",true);e("theme","default",function(e){As(e);$i(e)},true);e("keyMap","default",function(e,t,n){var r=Va(t);var i=n!=Is&&Va(n);if(i&&i.detach){i.detach(e,r)}if(r.attach){r.attach(e,i||null)}});e("extraKeys",null);e("configureMouse",null);e("lineWrapping",false,Ms,true);e("gutters",[],function(e,t){e.display.gutterSpecs=Xi(t,e.options.lineNumbers);$i(e)},true);e("fixedGutter",true,function(e,t){e.display.gutters.style.left=t?Nr(e.display)+"px":"0";e.refresh()},true);e("coverGutterNextToScrollbar",false,function(e){return gi(e)},true);e("scrollbarStyle","native",function(e){wi(e);gi(e);e.display.scrollbars.setScrollTop(e.doc.scrollTop);e.display.scrollbars.setScrollLeft(e.doc.scrollLeft)},true);e("lineNumbers",false,function(e,t){e.display.gutterSpecs=Xi(e.options.gutters,t);$i(e)},true);e("firstLineNumber",1,$i,true);e("lineNumberFormatter",function(e){return e},$i,true);e("showCursorWhenSelecting",false,zr,true);e("resetSelectionOnContextMenu",true);e("lineWiseCopyCut",true);e("pasteLinesPerSelection",true);e("selectionsMayTouch",false);e("readOnly",false,function(e,t){if(t=="nocursor"){Qr(e);e.display.input.blur()}e.display.input.readOnlyChanged(t)});e("screenReaderLabel",null,function(e,t){t=t===""?null:t;e.display.input.screenReaderLabelChanged(t)});e("disableInput",false,function(e,t){if(!t){e.display.input.reset()}},true);e("dragDrop",true,Ps);e("allowDropFileTypes",null);e("cursorBlinkRate",530);e("cursorScrollMargin",0);e("cursorHeight",1,zr,true);e("singleCursorHeightPerLine",true,zr,true);e("workTime",100);e("workDelay",100);e("flattenSpans",true,ho,true);e("addModeClass",false,ho,true);e("pollInterval",100);e("undoDepth",200,function(e,t){return e.doc.history.undoDepth=t});e("historyEventDelay",1250);e("viewportMargin",10,function(e){return e.refresh()},true);e("maxHighlightLength",1e4,ho,true);e("moveInputWithCursor",true,function(e,t){if(!t){e.display.input.resetPosition()}});e("tabindex",null,function(e,t){return e.display.input.getField().tabIndex=t||""});e("autofocus",null);e("direction","ltr",function(e,t){return e.doc.setDirection(t)},true);e("phrases",null)}function Ps(e,t,n){var r=n&&n!=Is;if(!t!=!r){var i=e.display.dragFunctions;var o=t?he:me;o(e.display.scroller,"dragstart",i.start);o(e.display.scroller,"dragenter",i.enter);o(e.display.scroller,"dragover",i.over);o(e.display.scroller,"dragleave",i.leave);o(e.display.scroller,"drop",i.drop)}}function Ms(e){if(e.options.lineWrapping){R(e.display.wrapper,"CodeMirror-wrap");e.display.sizer.style.minWidth="";e.display.sizerWidth=null}else{O(e.display.wrapper,"CodeMirror-wrap");sn(e)}Rr(e);Dr(e);fr(e);setTimeout(function(){return gi(e)},100)}function Ds(e,t){var n=this;if(!(this instanceof Ds)){return new Ds(e,t)}this.options=t=t?j(t):{};j(Ns,t,false);var r=t.value;if(typeof r=="string"){r=new xa(r,t.mode,null,t.lineSeparator,t.direction)}else if(t.mode){r.modeOption=t.mode}this.doc=r;var i=new Ds.inputStyles[t.inputStyle](this);var o=this.display=new Qi(e,r,i,t);o.wrapper.CodeMirror=this;As(this);if(t.lineWrapping){this.display.wrapper.className+=" CodeMirror-wrap"}wi(this);this.state={keyMaps:[],overlays:[],modeGen:0,overwrite:false,delayingBlurEvent:false,focused:false,suppressEdits:false,pasteIncoming:-1,cutIncoming:-1,selectingText:false,draggingText:false,highlight:new H,keySeq:null,specialChars:null};if(t.autofocus&&!d){o.input.focus()}if(E&&x<11){setTimeout(function(){return n.display.input.reset(true)},20)}js(this);Ia();xi(this);this.curOp.forceUpdate=true;yo(this,r);if(t.autofocus&&!d||this.hasFocus()){setTimeout(D($r,this),20)}else{Qr(this)}for(var a in Ls){if(Ls.hasOwnProperty(a)){Ls[a](this,t[a],Is)}}Yi(this);if(t.finishInit){t.finishInit(this)}for(var s=0;s20*20}he(o.scroller,"touchstart",function(e){if(!ye(i,e)&&!s(e)&&!Cs(i,e)){o.input.ensurePolled();clearTimeout(n);var t=+new Date;o.activeTouch={start:t,moved:false,prev:t-r.end<=300?r:null};if(e.touches.length==1){o.activeTouch.left=e.touches[0].pageX;o.activeTouch.top=e.touches[0].pageY}}});he(o.scroller,"touchmove",function(){if(o.activeTouch){o.activeTouch.moved=true}});he(o.scroller,"touchend",function(e){var t=o.activeTouch;if(t&&!Bn(o,e)&&t.left!=null&&!t.moved&&new Date-t.start<300){var n=i.coordsChar(o.activeTouch,"page"),r;if(!t.prev||l(t,t.prev)){r=new io(n,n)}else if(!t.prev.prev||l(t,t.prev.prev)){r=i.findWordAt(n)}else{r=new io(it(n.line,0),ft(i.doc,it(n.line+1,0)))}i.setSelection(r.anchor,r.head);i.focus();xe(e)}a()});he(o.scroller,"touchcancel",a);he(o.scroller,"scroll",function(){if(o.scroller.clientHeight){fi(i,o.scroller.scrollTop);pi(i,o.scroller.scrollLeft,true);ge(i,"scroll",i)}});he(o.scroller,"mousewheel",function(e){return no(i,e)});he(o.scroller,"DOMMouseScroll",function(e){return no(i,e)});he(o.wrapper,"scroll",function(){return o.wrapper.scrollTop=o.wrapper.scrollLeft=0});o.dragFunctions={enter:function(e){if(!ye(i,e)){Oe(e)}},over:function(e){if(!ye(i,e)){Ca(i,e);Oe(e)}},start:function(e){return Oa(i,e)},drop:Ni(i,Ta),leave:function(e){if(!ye(i,e)){ka(i)}}};var e=o.input.getField();he(e,"keyup",function(e){return cs.call(i,e)});he(e,"keydown",Ni(i,ls));he(e,"keypress",Ni(i,fs));he(e,"focus",function(e){return $r(i,e)});he(e,"blur",function(e){return Qr(i,e)})}Ds.defaults=Ns,Ds.optionHandlers=Ls;var Fs=[];function Hs(e,t,n,r){var i=e.doc,o;if(n==null){n="add"}if(n=="smart"){if(!i.mode.indent){n="prev"}else{o=yt(e,t).state}}var a=e.options.tabSize;var s=$e(i,t),l=F(s.text,null,a);if(s.stateAfter){s.stateAfter=null}var u=s.text.match(/^\s*/)[0],c;if(!r&&!/\S/.test(s.text)){c=0;n="not"}else if(n=="smart"){c=i.mode.indent(o,s.text.slice(u.length),s.text);if(c==z||c>150){if(!r){return}n="prev"}}if(n=="prev"){if(t>i.first){c=F($e(i,t-1).text,null,a)}else{c=0}}else if(n=="add"){c=l+e.options.indentUnit}else if(n=="subtract"){c=l-e.options.indentUnit}else if(typeof n=="number"){c=l+n}c=Math.max(0,c);var f="",d=0;if(e.options.indentWithTabs){for(var p=Math.floor(c/a);p;--p){d+=a;f+="\t"}}if(da;var l=Re(t),u=null;if(s&&r.ranges.length>1){if(Ws&&Ws.text.join("\n")==t){if(r.ranges.length%Ws.text.length==0){u=[];for(var c=0;c=0;d--){var p=r.ranges[d];var h=p.from(),v=p.to();if(p.empty()){if(n&&n>0){h=it(h.line,h.ch-n)}else if(e.state.overwrite&&!s){v=it(v.line,Math.min($e(o,v.line).text.length,v.ch+K(l).length))}else if(s&&Ws&&Ws.lineWise&&Ws.text.join("\n")==t){h=v=it(h.line,0)}}var m={from:h,to:v,text:u?u[d%u.length]:l,origin:i||(s?"paste":e.state.cutIncoming>a?"cut":"+input")};Qo(e.doc,m);_n(e,"inputRead",e,m)}if(t&&!s){Vs(e,t)}ai(e);if(e.curOp.updateInput<2){e.curOp.updateInput=f}e.curOp.typing=true;e.state.pasteIncoming=e.state.cutIncoming=-1}function Bs(e,t){var n=e.clipboardData&&e.clipboardData.getData("Text");if(n){e.preventDefault();if(!t.isReadOnly()&&!t.options.disableInput){Ii(t,function(){return zs(t,n,0,null,"paste")})}return true}}function Vs(e,t){if(!e.options.electricChars||!e.options.smartIndent){return}var n=e.doc.sel;for(var r=n.ranges.length-1;r>=0;r--){var i=n.ranges[r];if(i.head.ch>100||r&&n.ranges[r-1].head.line==i.head.line){continue}var o=e.getModeAt(i.head);var a=false;if(o.electricChars){for(var s=0;s-1){a=Hs(e,i.head.line,"smart");break}}}else if(o.electricInput){if(o.electricInput.test($e(e.doc,i.head.line).text.slice(0,i.head.ch))){a=Hs(e,i.head.line,"smart")}}if(a){_n(e,"electricInput",e,i.head.line)}}}function Gs(e){var t=[],n=[];for(var r=0;r0){jo(this.doc,r,new io(o,u[r].to()),B)}}else if(i.head.line>n){Hs(this,i.head.line,e,true);n=i.head.line;if(r==this.doc.sel.primIndex){ai(this)}}}}),getTokenAt:function(e,t){return St(this,e,t)},getLineTokens:function(e,t){return St(this,it(e),t,true)},getTokenTypeAt:function(e){e=ft(this.doc,e);var t=gt(this,$e(this.doc,e.line));var n=0,r=(t.length-1)/2,i=e.ch;var o;if(i==0){o=t[2]}else{for(;;){var a=n+r>>1;if((a?t[a*2-1]:0)>=i){r=a}else if(t[a*2+1]o){e=o;r=true}i=$e(this.doc,e)}else{i=e}return vr(this,i,{top:0,left:0},t||"page",n||r).top+(r?this.doc.height-on(i):0)},defaultTextHeight:function(){return _r(this.display)},defaultCharWidth:function(){return Ar(this.display)},getViewport:function(){return{from:this.display.viewFrom,to:this.display.viewTo}},addWidget:function(e,t,n,r,i){var o=this.display;e=yr(this,ft(this.doc,e));var a=e.bottom,s=e.left;t.style.position="absolute";t.setAttribute("cm-ignore-events","true");this.display.input.setUneditable(t);o.sizer.appendChild(t);if(r=="over"){a=e.top}else if(r=="above"||r=="near"){var l=Math.max(o.wrapper.clientHeight,this.doc.height),u=Math.max(o.sizer.clientWidth,o.lineSpace.clientWidth);if((r=="above"||e.bottom+t.offsetHeight>l)&&e.top>t.offsetHeight){a=e.top-t.offsetHeight}else if(e.bottom+t.offsetHeight<=l){a=e.bottom}if(s+t.offsetWidth>u){s=u-t.offsetWidth}}t.style.top=a+"px";t.style.left=t.style.right="";if(i=="right"){s=o.sizer.clientWidth-t.offsetWidth;t.style.right="0px"}else{if(i=="left"){s=0}else if(i=="middle"){s=(o.sizer.clientWidth-t.offsetWidth)/2}t.style.left=s+"px"}if(n){ri(this,{left:s,top:a,right:s+t.offsetWidth,bottom:a+t.offsetHeight})}},triggerOnKeyDown:Li(ls),triggerOnKeyPress:Li(fs),triggerOnKeyUp:cs,triggerOnMouseDown:Li(gs),execCommand:function(e){if($a.hasOwnProperty(e)){return $a[e].call(null,this)}},triggerElectric:Li(function(e){Vs(this,e)}),findPosH:function(e,t,n,r){var i=1;if(t<0){i=-1;t=-t}var o=ft(this.doc,e);for(var a=0;a0&&s(n.charAt(r-1))){--r}while(i.5||this.options.lineWrapping){Rr(this)}ge(this,"refresh",this)}),swapDoc:Li(function(e){var t=this.doc;t.cm=null;if(this.state.selectingText){this.state.selectingText()}yo(this,e);fr(this);this.display.input.reset();si(this,e.scrollLeft,e.scrollTop);this.curOp.forceScroll=true;_n(this,"swapDoc",this,t);return t}),phrase:function(e){var t=this.options.phrases;return t&&Object.prototype.hasOwnProperty.call(t,e)?t[e]:e},getInputField:function(){return this.display.input.getField()},getWrapperElement:function(){return this.display.wrapper},getScrollerElement:function(){return this.display.scroller},getGutterElement:function(){return this.display.gutters}};Ee(i);i.registerHelper=function(e,t,n){if(!u.hasOwnProperty(e)){u[e]=i[e]={_global:[]}}u[e][t]=n};i.registerGlobalHelper=function(e,t,n,r){i.registerHelper(e,t,r);u[e]._global.push({pred:n,val:r})}}function Ks(n,r,i,e,o){var t=r;var a=i;var s=$e(n,r.line);var l=o&&n.direction=="rtl"?-i:i;function u(){var e=r.line+l;if(e=n.first+n.size){return false}r=new it(e,r.ch,r.sticky);return s=$e(n,e)}function c(e){var t;if(o){t=Ka(n.cm,s,r,i)}else{t=Ya(s,r,i)}if(t==null){if(!e&&u()){r=Xa(o,n.cm,s,r.line,l)}else{return false}}else{r=t}return true}if(e=="char"){c()}else if(e=="column"){c(true)}else if(e=="word"||e=="group"){var f=null,d=e=="group";var p=n.cm&&n.cm.getHelper(r,"wordChars");for(var h=true;;h=false){if(i<0&&!c(!h)){break}var v=s.text.charAt(r.ch)||"\n";var m=ne(v,p)?"w":d&&v=="\n"?"n":!d||/\s/.test(v)?null:"p";if(d&&!h&&!m){m="s"}if(f&&f!=m){if(i<0){i=1;c();r.sticky="after"}break}if(m){f=m}if(i>0&&!c(!h)){break}}}var g=Yo(n,r,t,a,true);if(at(t,g)){g.hitSide=true}return g}function $s(e,t,n,r){var i=e.doc,o=t.left,a;if(r=="page"){var s=Math.min(e.display.wrapper.clientHeight,window.innerHeight||document.documentElement.clientHeight);var l=Math.max(s-.5*_r(e.display),3);a=(n>0?t.bottom:t.top)+n*l}else if(r=="line"){a=n>0?t.bottom+3:t.top-3}var u;for(;;){u=Er(e,o,a);if(!u.outside){break}if(n<0?a<=0:a>=i.height){u.hitSide=true;break}a+=n*5}return u}var Qs=function(e){this.cm=e;this.lastAnchorNode=this.lastAnchorOffset=this.lastFocusNode=this.lastFocusOffset=null;this.polling=new H;this.composing=null;this.gracePeriod=false;this.readDOMTimeout=null};function Js(e,t){var n=er(e,t.line);if(!n||n.hidden){return null}var r=$e(e.doc,t.line);var i=Qn(n,r,t.line);var o=de(r,e.doc.direction),a="left";if(o){var s=ce(o,t.ch);a=s%2?"right":"left"}var l=or(i.map,t.ch,a);l.offset=l.collapse=="right"?l.end:l.start;return l}function Zs(e){for(var t=e;t;t=t.parentNode){if(/CodeMirror-gutter-wrapper/.test(t.className)){return true}}return false}function el(e,t){if(t){e.bad=true}return e}function tl(s,e,t,l,u){var n="",c=false,f=s.doc.lineSeparator(),d=false;function p(t){return function(e){return e.id==t}}function h(){if(c){n+=f;if(d){n+=f}c=d=false}}function v(e){if(e){h();n+=e}}function m(e){if(e.nodeType==1){var t=e.getAttribute("cm-text");if(t){v(t);return}var n=e.getAttribute("cm-marker"),r;if(n){var i=s.findMarks(it(l,0),it(u+1,0),p(+n));if(i.length&&(r=i[0].find(0))){v(Qe(s.doc,r.from,r.to).join(f))}return}if(e.getAttribute("contenteditable")=="false"){return}var o=/^(pre|div|p|li|table|br)$/i.test(e.nodeName);if(!/^br$/i.test(e.nodeName)&&e.textContent.length==0){return}if(o){h()}for(var a=0;a=t.display.viewTo||i.line=t.display.viewFrom&&Js(t,r)||{node:s[0].measure.map[2],offset:0};var u=i.linee.firstLine()){r=it(r.line-1,$e(e.doc,r.line-1).length)}if(i.ch==$e(e.doc,i.line).text.length&&i.linet.viewTo-1){return false}var o,a,s;if(r.line==t.viewFrom||(o=Mr(e,r.line))==0){a=et(t.view[0].line);s=t.view[0].node}else{a=et(t.view[o].line);s=t.view[o-1].node.nextSibling}var l=Mr(e,i.line);var u,c;if(l==t.view.length-1){u=t.viewTo-1;c=t.lineDiv.lastChild}else{u=et(t.view[l+1].line)-1;c=t.view[l+1].node.previousSibling}if(!s){return false}var f=e.doc.splitLines(tl(e,s,c,a,u));var d=Qe(e.doc,it(a,0),it(u,$e(e.doc,u).text.length));while(f.length>1&&d.length>1){if(K(f)==K(d)){f.pop();d.pop();u--}else if(f[0]==d[0]){f.shift();d.shift();a++}else{break}}var p=0,h=0;var v=f[0],m=d[0],g=Math.min(v.length,m.length);while(pr.ch&&y.charCodeAt(y.length-h-1)==b.charCodeAt(b.length-h-1)){p--;h++}}f[f.length-1]=y.slice(0,y.length-h).replace(/^\u200b+/,"");f[0]=f[0].slice(p).replace(/\u200b+$/,"");var E=it(a,p);var x=it(u,d.length?K(d).length-h:0);if(f.length>1||f[0]||ot(E,x)){ra(e.doc,f,E,x,"+input");return true}},Qs.prototype.ensurePolled=function(){this.forceCompositionEnd()},Qs.prototype.reset=function(){this.forceCompositionEnd()},Qs.prototype.forceCompositionEnd=function(){if(!this.composing){return}clearTimeout(this.readDOMTimeout);this.composing=null;this.updateFromDOM();this.div.blur();this.div.focus()},Qs.prototype.readFromDOMSoon=function(){var e=this;if(this.readDOMTimeout!=null){return}this.readDOMTimeout=setTimeout(function(){e.readDOMTimeout=null;if(e.composing){if(e.composing.done){e.composing=null}else{return}}e.updateFromDOM()},80)},Qs.prototype.updateFromDOM=function(){var e=this;if(this.cm.isReadOnly()||!this.pollContent()){Ii(this.cm,function(){return Dr(e.cm)})}},Qs.prototype.setUneditable=function(e){e.contentEditable="false"},Qs.prototype.onKeyPress=function(e){if(e.charCode==0||this.composing){return}e.preventDefault();if(!this.cm.isReadOnly()){Ni(this.cm,zs)(this.cm,String.fromCharCode(e.charCode==null?e.keyCode:e.charCode),0)}},Qs.prototype.readOnlyChanged=function(e){this.div.contentEditable=String(e!="nocursor")},Qs.prototype.onContextMenu=function(){},Qs.prototype.resetPosition=function(){},Qs.prototype.needsContentAttribute=true;var il=function(e){this.cm=e;this.prevInput="";this.pollingFast=false;this.polling=new H;this.hasSelection=false;this.composing=null};function ol(t,n){if((n=n?j(n):{}).value=t.value,!n.tabindex&&t.tabIndex)n.tabindex=t.tabIndex;if(!n.placeholder&&t.placeholder)n.placeholder=t.placeholder;if(null==n.autofocus){var e=L();n.autofocus=e==t||null!=t.getAttribute("autofocus")&&e==document.body}function r(){t.value=s.getValue()}var i;if(t.form)if(he(t.form,"submit",r),!n.leaveSubmitMethodAlone){var o=t.form;i=o.submit;try{var a=o.submit=function(){r(),o.submit=i,o.submit(),o.submit=a}}catch(e){}}n.finishInit=function(e){e.save=r,e.getTextArea=function(){return t},e.toTextArea=function(){e.toTextArea=isNaN,r(),t.parentNode.removeChild(e.getWrapperElement()),t.style.display="",t.form&&(me(t.form,"submit",r),n.leaveSubmitMethodAlone||"function"!=typeof t.form.submit||(t.form.submit=i))}},t.style.display="none";var s=Ds(function(e){return t.parentNode.insertBefore(e,t.nextSibling)},n);return s}function al(e){e.off=me,e.on=he,e.wheelEventPixels=to,e.Doc=xa,e.splitLines=Re,e.countColumn=F,e.findColumn=q,e.isWordChar=te,e.Pass=z,e.signal=ge,e.Line=ln,e.changeEnd=so,e.scrollbarModel=bi,e.Pos=it,e.cmpPos=ot,e.modes=Fe,e.mimeModes=He,e.resolveMode=ze,e.getMode=Be,e.modeExtensions=Ve,e.extendMode=Ge,e.copyState=qe,e.startState=Xe,e.innerMode=Ye,e.commands=$a,e.keyMap=ja,e.keyName=Ba,e.isModifierKey=Ua,e.lookupKey=Wa,e.normalizeKeyMap=Ha,e.StringStream=Ke,e.SharedTextMarker=ma,e.TextMarker=ha,e.LineWidget=ca,e.e_preventDefault=xe,e.e_stopPropagation=Se,e.e_stop=Oe,e.addClass=R,e.contains=I,e.rmClass=O,e.keyNames=Ra}il.prototype.init=function(n){var e=this;var r=this,i=this.cm;this.createField(n);var o=this.textarea;n.wrapper.insertBefore(this.wrapper,n.wrapper.firstChild);if(c){o.style.width="0px"}he(o,"input",function(){if(E&&x>=9&&e.hasSelection){e.hasSelection=null}r.poll()});he(o,"paste",function(e){if(ye(i,e)||Bs(e,i)){return}i.state.pasteIncoming=+new Date;r.fastPoll()});function t(e){if(ye(i,e)){return}if(i.somethingSelected()){Us({lineWise:false,text:i.getSelections()})}else if(!i.options.lineWiseCopyCut){return}else{var t=Gs(i);Us({lineWise:true,text:t.text});if(e.type=="cut"){i.setSelections(t.ranges,null,B)}else{r.prevInput="";o.value=t.text.join("\n");M(o)}}if(e.type=="cut"){i.state.cutIncoming=+new Date}}he(o,"cut",t);he(o,"copy",t);he(n.scroller,"paste",function(e){if(Bn(n,e)||ye(i,e)){return}if(!o.dispatchEvent){i.state.pasteIncoming=+new Date;r.focus();return}var t=new Event("paste");t.clipboardData=e.clipboardData;o.dispatchEvent(t)});he(n.lineSpace,"selectstart",function(e){if(!Bn(n,e)){xe(e)}});he(o,"compositionstart",function(){var e=i.getCursor("from");if(r.composing){r.composing.range.clear()}r.composing={start:e,range:i.markText(e,i.getCursor("to"),{className:"CodeMirror-composing"})}});he(o,"compositionend",function(){if(r.composing){r.poll();r.composing.range.clear();r.composing=null}})},il.prototype.createField=function(e){this.wrapper=Ys();this.textarea=this.wrapper.firstChild},il.prototype.screenReaderLabelChanged=function(e){if(e){this.textarea.setAttribute("aria-label",e)}else{this.textarea.removeAttribute("aria-label")}},il.prototype.prepareSelection=function(){var e=this.cm,t=e.display,n=e.doc;var r=Br(e);if(e.options.moveInputWithCursor){var i=yr(e,n.sel.primary().head,"div");var o=t.wrapper.getBoundingClientRect(),a=t.lineDiv.getBoundingClientRect();r.teTop=Math.max(0,Math.min(t.wrapper.clientHeight-10,i.top+a.top-o.top));r.teLeft=Math.max(0,Math.min(t.wrapper.clientWidth-10,i.left+a.left-o.left))}return r},il.prototype.showSelection=function(e){var t=this.cm,n=t.display;_(n.cursorDiv,e.cursors);_(n.selectionDiv,e.selection);if(e.teTop!=null){this.wrapper.style.top=e.teTop+"px";this.wrapper.style.left=e.teLeft+"px"}},il.prototype.reset=function(e){if(this.contextMenuPending||this.composing){return}var t=this.cm;if(t.somethingSelected()){this.prevInput="";var n=t.getSelection();this.textarea.value=n;if(t.state.focused){M(this.textarea)}if(E&&x>=9){this.hasSelection=n}}else if(!e){this.prevInput=this.textarea.value="";if(E&&x>=9){this.hasSelection=null}}},il.prototype.getField=function(){return this.textarea},il.prototype.supportsTouch=function(){return false},il.prototype.focus=function(){if(this.cm.options.readOnly!="nocursor"&&(!d||L()!=this.textarea)){try{this.textarea.focus()}catch(e){}}},il.prototype.blur=function(){this.textarea.blur()},il.prototype.resetPosition=function(){this.wrapper.style.top=this.wrapper.style.left=0},il.prototype.receivedFocus=function(){this.slowPoll()},il.prototype.slowPoll=function(){var e=this;if(this.pollingFast){return}this.polling.set(this.cm.options.pollInterval,function(){e.poll();if(e.cm.state.focused){e.slowPoll()}})},il.prototype.fastPoll=function(){var t=false,n=this;n.pollingFast=true;function r(){var e=n.poll();if(!e&&!t){t=true;n.polling.set(60,r)}else{n.pollingFast=false;n.slowPoll()}}n.polling.set(20,r)},il.prototype.poll=function(){var e=this;var t=this.cm,n=this.textarea,r=this.prevInput;if(this.contextMenuPending||!t.state.focused||Pe(n)&&!r&&!this.composing||t.isReadOnly()||t.options.disableInput||t.state.keySeq){return false}var i=n.value;if(i==r&&!t.somethingSelected()){return false}if(E&&x>=9&&this.hasSelection===i||y&&/[\uf700-\uf7ff]/.test(i)){t.display.input.reset();return false}if(t.doc.sel==t.display.selForContextMenu){var o=i.charCodeAt(0);if(o==8203&&!r){r="​"}if(o==8666){this.reset();return this.cm.execCommand("undo")}}var a=0,s=Math.min(r.length,i.length);while(a1e3||i.indexOf("\n")>-1){n.value=e.prevInput=""}else{e.prevInput=i}if(e.composing){e.composing.range.clear();e.composing.range=t.markText(e.composing.start,t.getCursor("to"),{className:"CodeMirror-composing"})}});return true},il.prototype.ensurePolled=function(){if(this.pollingFast&&this.poll()){this.pollingFast=false}},il.prototype.onKeyPress=function(){if(E&&x>=9){this.hasSelection=null}this.fastPoll()},il.prototype.onContextMenu=function(e){var n=this,r=n.cm,i=r.display,o=n.textarea;if(n.contextMenuPending){n.contextMenuPending()}var t=Pr(r,e),a=i.scroller.scrollTop;if(!t||g){return}var s=r.options.resetSelectionOnContextMenu;if(s&&r.doc.sel.contains(t)==-1){Ni(r,Uo)(r.doc,ao(t),B)}var l=o.style.cssText,u=n.wrapper.style.cssText;var c=n.wrapper.offsetParent.getBoundingClientRect();n.wrapper.style.cssText="position: static";o.style.cssText="position: absolute; width: 30px; height: 30px;\n top: "+(e.clientY-c.top-5)+"px; left: "+(e.clientX-c.left-5)+"px;\n z-index: 1000; background: "+(E?"rgba(255, 255, 255, .05)":"transparent")+";\n outline: none; border-width: 0; outline: none; overflow: hidden; opacity: .05; filter: alpha(opacity=5);";var f;if(m){f=window.scrollY}i.input.focus();if(m){window.scrollTo(null,f)}i.input.reset();if(!r.somethingSelected()){o.value=n.prevInput=" "}n.contextMenuPending=p;i.selForContextMenu=r.doc.sel;clearTimeout(i.detectingSelectAll);function d(){if(o.selectionStart!=null){var e=r.somethingSelected();var t="​"+(e?o.value:"");o.value="⇚";o.value=t;n.prevInput=e?"":"​";o.selectionStart=1;o.selectionEnd=t.length;i.selForContextMenu=r.doc.sel}}function p(){if(n.contextMenuPending!=p){return}n.contextMenuPending=false;n.wrapper.style.cssText=u;o.style.cssText=l;if(E&&x<9){i.scrollbars.setScrollTop(i.scroller.scrollTop=a)}if(o.selectionStart!=null){if(!E||E&&x<9){d()}var e=0,t=function(){if(i.selForContextMenu==r.doc.sel&&o.selectionStart==0&&o.selectionEnd>0&&n.prevInput=="​"){Ni(r,Ko)(r)}else if(e++<10){i.detectingSelectAll=setTimeout(t,500)}else{i.selForContextMenu=null;i.input.reset()}};i.detectingSelectAll=setTimeout(t,200)}}if(E&&x>=9){d()}if(S){Oe(e);var h=function(){me(window,"mouseup",h);setTimeout(p,20)};he(window,"mouseup",h)}else{setTimeout(p,50)}},il.prototype.readOnlyChanged=function(e){if(!e){this.reset()}this.textarea.disabled=e=="nocursor"},il.prototype.setUneditable=function(){},il.prototype.needsContentAttribute=false,Rs(Ds),Xs(Ds);var sl="iter insert remove copy getEditor constructor".split(" ");for(var ll in xa.prototype){if(xa.prototype.hasOwnProperty(ll)&&W(sl,ll)<0){Ds.prototype[ll]=function(e){return function(){return e.apply(this.doc,arguments)}}(xa.prototype[ll])}}return Ee(xa),Ds.inputStyles={textarea:il,contenteditable:Qs},Ds.defineMode=function(e){Ds.defaults.mode||"null"==e||(Ds.defaults.mode=e),function(e,t){2i;)c(Y,t=n[i++])||t==B||t==h||r.push(t);return r}function l(e){for(var t,n=e===K,r=F(n?X:k(e)),i=[],o=0;r.length>o;)!c(Y,t=r[o++])||n&&!c(K,t)||i.push(Y[t]);return i}var u=n(6),c=n(23),f=n(12),d=n(1),p=n(25),h=n(49).KEY,v=n(8),m=n(67),g=n(63),y=n(54),b=n(10),w=n(138),E=n(97),x=n(203),S=n(78),T=n(4),O=n(7),C=n(17),k=n(21),_=n(38),A=n(47),I=n(44),N=n(141),L=n(26),R=n(77),P=n(15),M=n(43),D=L.f,j=P.f,F=N.f,H=u.Symbol,W=u.JSON,U=W&&W.stringify,z="prototype",B=b("_hidden"),V=b("toPrimitive"),G={}.propertyIsEnumerable,q=m("symbol-registry"),Y=m("symbols"),X=m("op-symbols"),K=Object[z],$="function"==typeof H&&!!R.f,Q=u.QObject,J=!Q||!Q[z]||!Q[z].findChild,Z=f&&v(function(){return 7!=I(j({},"a",{get:function(){return j(this,"a",{value:7}).a}})).a})?function(e,t,n){var r=D(K,t);r&&delete K[t],j(e,t,n),r&&e!==K&&j(K,t,r)}:j,ee=$&&"symbol"==typeof H.iterator?function(e){return"symbol"==typeof e}:function(e){return e instanceof H},te=function(e,t,n){return e===K&&te(X,t,n),T(e),t=_(t,!0),T(n),c(Y,t)?(n.enumerable?(c(e,B)&&e[B][t]&&(e[B][t]=!1),n=I(n,{enumerable:A(0,!1)})):(c(e,B)||j(e,B,A(1,{})),e[B][t]=!0),Z(e,t,n)):j(e,t,n)};$||(p((H=function(e){if(this instanceof H)throw TypeError("Symbol is not a constructor!");var t=y(0re;)b(ne[re++]);for(var ie=M(b.store),oe=0;ie.length>oe;)E(ie[oe++]);d(d.S+d.F*!$,"Symbol",{for:function(e){return c(q,e+="")?q[e]:q[e]=H(e)},keyFor:function(e){if(!ee(e))throw TypeError(e+" is not a symbol!");for(var t in q)if(q[t]===e)return t},useSetter:function(){J=!0},useSimple:function(){J=!1}}),d(d.S+d.F*!$,"Object",{create:function(e,t){return void 0===t?I(e):i(I(e),t)},defineProperty:te,defineProperties:i,getOwnPropertyDescriptor:a,getOwnPropertyNames:s,getOwnPropertySymbols:l});var ae=v(function(){R.f(1)});d(d.S+d.F*ae,"Object",{getOwnPropertySymbols:function(e){return R.f(C(e))}}),W&&d(d.S+d.F*(!$||v(function(){var e=H();return"[null]"!=U([e])||"{}"!=U({a:e})||"{}"!=U(Object(e))})),"JSON",{stringify:function(e){for(var t,n,r=[e],i=1;ia;)o.call(e,r=i[a++])&&t.push(r);return t}},function(e,t,n){var r=n(1);r(r.S,"Object",{create:n(44)})},function(e,t,n){var r=n(1);r(r.S+r.F*!n(12),"Object",{defineProperty:n(15).f})},function(e,t,n){var r=n(1);r(r.S+r.F*!n(12),"Object",{defineProperties:n(140)})},function(e,t,n){var r=n(21),i=n(26).f;n(40)("getOwnPropertyDescriptor",function(){return function(e,t){return i(r(e),t)}})},function(e,t,n){var r=n(17),i=n(27);n(40)("getPrototypeOf",function(){return function(e){return i(r(e))}})},function(e,t,n){var r=n(17),i=n(43);n(40)("keys",function(){return function(e){return i(r(e))}})},function(e,t,n){n(40)("getOwnPropertyNames",function(){return n(141).f})},function(e,t,n){var r=n(7),i=n(49).onFreeze;n(40)("freeze",function(t){return function(e){return t&&r(e)?t(i(e)):e}})},function(e,t,n){var r=n(7),i=n(49).onFreeze;n(40)("seal",function(t){return function(e){return t&&r(e)?t(i(e)):e}})},function(e,t,n){var r=n(7),i=n(49).onFreeze;n(40)("preventExtensions",function(t){return function(e){return t&&r(e)?t(i(e)):e}})},function(e,t,n){var r=n(7);n(40)("isFrozen",function(t){return function(e){return!r(e)||!!t&&t(e)}})},function(e,t,n){var r=n(7);n(40)("isSealed",function(t){return function(e){return!r(e)||!!t&&t(e)}})},function(e,t,n){var r=n(7);n(40)("isExtensible",function(t){return function(e){return!!r(e)&&(!t||t(e))}})},function(e,t,n){var r=n(1);r(r.S+r.F,"Object",{assign:n(101)})},function(e,t,n){var r=n(1);r(r.S,"Object",{is:n(142)})},function(e,t,n){var r=n(1);r(r.S,"Object",{setPrototypeOf:n(102).set})},function(e,t,n){"use strict";var r=n(50),i={};i[n(10)("toStringTag")]="z",i+""!="[object z]"&&n(25)(Object.prototype,"toString",function(){return"[object "+r(this)+"]"},!0)},function(e,t,n){var r=n(1);r(r.P,"Function",{bind:n(143)})},function(e,t,n){var r=n(15).f,i=Function.prototype,o=/^\s*function ([^ (]*)/;"name"in i||n(12)&&r(i,"name",{configurable:!0,get:function(){try{return(""+this).match(o)[1]}catch(e){return""}}})},function(e,t,n){"use strict";var r=n(7),i=n(27),o=n(10)("hasInstance"),a=Function.prototype;o in a||n(15).f(a,o,{value:function(e){if("function"!=typeof this||!r(e))return!1;if(!r(this.prototype))return e instanceof this;for(;e=i(e);)if(this.prototype===e)return!0;return!1}})},function(e,t,n){var r=n(1),i=n(144);r(r.G+r.F*(parseInt!=i),{parseInt:i})},function(e,t,n){var r=n(1),i=n(145);r(r.G+r.F*(parseFloat!=i),{parseFloat:i})},function(e,t,n){"use strict";function r(e){var t=c(e,!1);if("string"==typeof t&&2x;x++)o(m,w=E[x])&&!o(v,w)&&d(v,w,f(m,w));(v.prototype=g).constructor=v,n(25)(i,h,v)}},function(e,t,n){"use strict";function u(e,t){for(var n=-1,r=t;++n<6;)r+=e*a[n],a[n]=r%1e7,r=o(r/1e7)}function c(e){for(var t=6,n=0;0<=--t;)n+=a[t],a[t]=o(n/e),n=n%e*1e7}function f(){for(var e=6,t="";0<=--e;)if(""!==t||0===e||0!==a[e]){var n=String(a[e]);t=""===t?n:t+h.call("0",7-n.length)+n}return t}var r=n(1),d=n(33),p=n(146),h=n(106),i=1..toFixed,o=Math.floor,a=[0,0,0,0,0,0],v="Number.toFixed: incorrect invocation!",m=function(e,t,n){return 0===t?n:t%2==1?m(e,t-1,n*e):m(e*e,t/2,n)};r(r.P+r.F*(!!i&&("0.000"!==8e-5.toFixed(3)||"1"!==.9.toFixed(0)||"1.25"!==1.255.toFixed(2)||"1000000000000000128"!==(0xde0b6b3a7640080).toFixed(0))||!n(8)(function(){i.call({})})),"Number",{toFixed:function(e){var t,n,r,i,o=p(this,v),a=d(e),s="",l="0";if(a<0||20>>=0)?31-Math.floor(Math.log(e+.5)*Math.LOG2E):32}})},function(e,t,n){var r=n(1),i=Math.exp;r(r.S,"Math",{cosh:function(e){return(i(e=+e)+i(-e))/2}})},function(e,t,n){var r=n(1),i=n(108);r(r.S+r.F*(i!=Math.expm1),"Math",{expm1:i})},function(e,t,n){var r=n(1);r(r.S,"Math",{fround:n(149)})},function(e,t,n){var r=n(1),l=Math.abs;r(r.S,"Math",{hypot:function(e,t){for(var n,r,i=0,o=0,a=arguments.length,s=0;o>>16)*o+i*(65535&r>>>16)<<16>>>0)}})},function(e,t,n){var r=n(1);r(r.S,"Math",{log10:function(e){return Math.log(e)*Math.LOG10E}})},function(e,t,n){var r=n(1);r(r.S,"Math",{log1p:n(148)})},function(e,t,n){var r=n(1);r(r.S,"Math",{log2:function(e){return Math.log(e)/Math.LN2}})},function(e,t,n){var r=n(1);r(r.S,"Math",{sign:n(107)})},function(e,t,n){var r=n(1),i=n(108),o=Math.exp;r(r.S+r.F*n(8)(function(){return-2e-17!=!Math.sinh(-2e-17)}),"Math",{sinh:function(e){return Math.abs(e=+e)<1?(i(e)-i(-e))/2:(o(e-1)-o(-e-1))*(Math.E/2)}})},function(e,t,n){var r=n(1),i=n(108),o=Math.exp;r(r.S,"Math",{tanh:function(e){var t=i(e=+e),n=i(-e);return t==1/0?1:n==1/0?-1:(t-n)/(o(e)+o(-e))}})},function(e,t,n){var r=n(1);r(r.S,"Math",{trunc:function(e){return(0>10),t%1024+56320))}return n.join("")}})},function(e,t,n){var r=n(1),a=n(21),s=n(13);r(r.S,"String",{raw:function(e){for(var t=a(e.raw),n=s(t.length),r=arguments.length,i=[],o=0;o=t.length?{value:void 0,done:!0}:(e=r(t,n),this._i+=e.length,{value:e,done:!1})})},function(e,t,n){"use strict";var r=n(1),i=n(79)(!1);r(r.P,"String",{codePointAt:function(e){return i(this,e)}})},function(e,t,n){"use strict";var r=n(1),s=n(13),l=n(109),u="endsWith",c=""[u];r(r.P+r.F*n(110)(u),"String",{endsWith:function(e,t){var n=l(this,e,u),r=1g;)v(m[g++]);(f.constructor=u).prototype=f,n(25)(r,"RegExp",u)}n(58)("RegExp")},function(e,t,n){"use strict";n(154);function r(e){n(25)(RegExp.prototype,s,e,!0)}var i=n(4),o=n(71),a=n(12),s="toString",l=/./[s];n(8)(function(){return"/a/b"!=l.call({source:"a",flags:"b"})})?r(function(){var e=i(this);return"/".concat(e.source,"/","flags"in e?e.flags:!a&&e instanceof RegExp?o.call(e):void 0)}):l.name!=s&&r(function(){return l.call(this)})},function(e,t,n){"use strict";var f=n(4),d=n(13),p=n(118),h=n(84);n(85)("match",1,function(r,i,u,c){return[function(e){var t=r(this),n=null==e?void 0:e[i];return void 0!==n?n.call(e,t):new RegExp(e)[i](String(t))},function(e){var t=c(u,e,this);if(t.done)return t.value;var n=f(e),r=String(this);if(!n.global)return h(n,r);for(var i,o=n.unicode,a=[],s=n.lastIndex=0;null!==(i=h(n,r));){var l=String(i[0]);""===(a[s]=l)&&(n.lastIndex=p(r,d(n.lastIndex),o)),s++}return 0===s?null:a}]})},function(e,t,n){"use strict";var T=n(4),r=n(17),O=n(13),C=n(33),k=n(118),_=n(84),A=Math.max,I=Math.min,d=Math.floor,p=/\$([$&`']|\d\d?|<[^>]*>)/g,h=/\$([$&`']|\d\d?)/g;n(85)("replace",2,function(i,o,E,x){return[function(e,t){var n=i(this),r=null==e?void 0:e[o];return void 0!==r?r.call(e,n,t):E.call(String(n),e,t)},function(e,t){var n=x(E,e,this,t);if(n.done)return n.value;var r=T(e),i=String(this),o="function"==typeof t;o||(t=String(t));var a=r.global;if(a){var s=r.unicode;r.lastIndex=0}for(var l=[];;){var u=_(r,i);if(null===u)break;if(l.push(u),!a)break;""===String(u[0])&&(r.lastIndex=k(i,O(r.lastIndex),s))}for(var c,f="",d=0,p=0;p>>0,c=new RegExp(e.source,s+"g");(r=d.call(c,n))&&!(l<(i=c[v])&&(a.push(n.slice(l,r.index)),1=u));)c[v]===r.index&&c[v]++;return l===n[h]?!o&&c.test("")||a.push(""):a.push(n.slice(l)),a[h]>u?a.slice(0,u):a}:"0"[a](void 0,0)[h]?function(e,t){return void 0===e&&0===t?[]:m.call(this,e,t)}:m,[function(e,t){var n=i(this),r=null==e?void 0:e[o];return void 0!==r?r.call(e,n,t):y.call(String(n),e,t)},function(e,t){var n=g(y,e,this,t,y!==m);if(n.done)return n.value;var r=b(e),i=String(this),o=w(r,RegExp),a=r.unicode,s=(r.ignoreCase?"i":"")+(r.multiline?"m":"")+(r.unicode?"u":"")+(C?"y":"g"),l=new o(C?r:"^(?:"+r.source+")",s),u=void 0===t?O:t>>>0;if(0==u)return[];if(0===i.length)return null===S(l,i)?[i]:[];for(var c=0,f=0,d=[];fe;)t(r[e++]);c._c=[],c._n=!1,n&&!c._h&&D(c)})}}function o(e){var t=this;t._d||(t._d=!0,(t=t._w||t)._v=e,t._s=2,t._a||(t._a=t._c.slice()),i(t,!0))}var a,s,l,u,c=n(48),d=n(6),p=n(29),h=n(50),v=n(1),m=n(7),g=n(19),y=n(59),b=n(52),w=n(72),E=n(119).set,x=n(120)(),S=n(121),T=n(155),O=n(86),C=n(156),k="Promise",_=d.TypeError,A=d.process,I=A&&A.versions,N=I&&I.v8||"",L=d[k],R="process"==h(A),P=s=S.f,M=!!function(){try{var e=L.resolve(1),t=(e.constructor={})[n(10)("species")]=function(e){e(r,r)};return(R||"function"==typeof PromiseRejectionEvent)&&e.then(r)instanceof t&&0!==N.indexOf("6.6")&&-1===O.indexOf("Chrome/66")}catch(e){}}(),D=function(o){E.call(d,function(){var e,t,n,r=o._v,i=j(o);if(i&&(e=T(function(){R?A.emit("unhandledRejection",r,o):(t=d.onunhandledrejection)?t({promise:o,reason:r}):(n=d.console)&&n.error&&n.error("Unhandled promise rejection",r)}),o._h=R||j(o)?2:1),o._a=void 0,i&&e.e)throw e.v})},j=function(e){return 1!==e._h&&0===(e._a||e._c).length},F=function(t){E.call(d,function(){var e;R?A.emit("rejectionHandled",t):(e=d.onrejectionhandled)&&e({promise:t,reason:t._v})})},H=function(e){var n,r=this;if(!r._d){r._d=!0,r=r._w||r;try{if(r===e)throw _("Promise can't be resolved itself");(n=f(e))?x(function(){var t={_w:r,_d:!1};try{n.call(e,p(H,t,1),p(o,t,1))}catch(e){o.call(t,e)}}):(r._v=e,r._s=1,i(r,!1))}catch(e){o.call({_w:r,_d:!1},e)}}};M||(L=function(e){y(this,L,k,"_h"),g(e),a.call(this);try{e(p(H,this,1),p(o,this,1))}catch(e){o.call(this,e)}},(a=function(){this._c=[],this._a=void 0,this._s=0,this._d=!1,this._v=void 0,this._h=0,this._n=!1}).prototype=n(60)(L.prototype,{then:function(e,t){var n=P(w(this,L));return n.ok="function"!=typeof e||e,n.fail="function"==typeof t&&t,n.domain=R?A.domain:void 0,this._c.push(n),this._a&&this._a.push(n),this._s&&i(this,!1),n.promise},catch:function(e){return this.then(void 0,e)}}),l=function(){var e=new a;this.promise=e,this.resolve=p(H,e,1),this.reject=p(o,e,1)},S.f=P=function(e){return e===L||e===u?new l(e):s(e)}),v(v.G+v.W+v.F*!M,{Promise:L}),n(63)(L,k),n(58)(k),u=n(20)[k],v(v.S+v.F*!M,k,{reject:function(e){var t=P(this);return(0,t.reject)(e),t.promise}}),v(v.S+v.F*(c||!M),k,{resolve:function(e){return C(c&&this===u?L:this,e)}}),v(v.S+v.F*!(M&&n(83)(function(e){L.all(e).catch(r)})),k,{all:function(e){var a=this,t=P(a),s=t.resolve,l=t.reject,n=T(function(){var r=[],i=0,o=1;b(e,!1,function(e){var t=i++,n=!1;r.push(void 0),o++,a.resolve(e).then(function(e){n||(n=!0,r[t]=e,--o||s(r))},l)}),--o||s(r)});return n.e&&l(n.v),t.promise},race:function(e){var t=this,n=P(t),r=n.reject,i=T(function(){b(e,!1,function(e){t.resolve(e).then(n.resolve,r)})});return i.e&&r(i.v),n.promise}})},function(e,t,n){"use strict";var r=n(161),i=n(61);n(87)("WeakSet",function(t){return function(e){return t(this,0=t.length)return{value:void 0,done:!0}}while(!((e=t[this._i++])in this._t));return{value:e,done:!1}}),i(i.S,"Reflect",{enumerate:function(e){return new r(e)}})},function(e,t,n){var a=n(26),s=n(27),l=n(23),r=n(1),u=n(7),c=n(4);r(r.S,"Reflect",{get:function e(t,n){var r,i,o=arguments.length<3?t:arguments[2];return c(t)===o?t[n]:(r=a.f(t,n))?l(r,"value")?r.value:void 0!==r.get?r.get.call(o):void 0:u(i=s(t))?e(i,n,o):void 0}})},function(e,t,n){var r=n(26),i=n(1),o=n(4);i(i.S,"Reflect",{getOwnPropertyDescriptor:function(e,t){return r.f(o(e),t)}})},function(e,t,n){var r=n(1),i=n(27),o=n(4);r(r.S,"Reflect",{getPrototypeOf:function(e){return i(o(e))}})},function(e,t,n){var r=n(1);r(r.S,"Reflect",{has:function(e,t){return t in e}})},function(e,t,n){var r=n(1),i=n(4),o=Object.isExtensible;r(r.S,"Reflect",{isExtensible:function(e){return i(e),!o||o(e)}})},function(e,t,n){var r=n(1);r(r.S,"Reflect",{ownKeys:n(123)})},function(e,t,n){var r=n(1),i=n(4),o=Object.preventExtensions;r(r.S,"Reflect",{preventExtensions:function(e){i(e);try{return o&&o(e),!0}catch(e){return!1}}})},function(e,t,n){var l=n(15),u=n(26),c=n(27),f=n(23),r=n(1),d=n(47),p=n(4),h=n(7);r(r.S,"Reflect",{set:function e(t,n,r){var i,o,a=arguments.length<4?t:arguments[3],s=u.f(p(t),n);if(!s){if(h(o=c(t)))return e(o,n,r,a);s=d(0)}if(f(s,"value")){if(!1===s.writable||!h(a))return!1;if(i=u.f(a,n)){if(i.get||i.set||!1===i.writable)return!1;i.value=r,l.f(a,n,i)}else l.f(a,n,d(0,r));return!0}return void 0!==s.set&&(s.set.call(a,r),!0)}})},function(e,t,n){var r=n(1),i=n(102);i&&r(r.S,"Reflect",{setPrototypeOf:function(e,t){i.check(e,t);try{return i.set(e,t),!0}catch(e){return!1}}})},function(e,t,n){"use strict";var r=n(1),i=n(76)(!0);r(r.P,"Array",{includes:function(e,t){return i(this,e,1s;)void 0!==(n=i(r,t=o[s++]))&&f(a,t,n);return a}})},function(e,t,n){var r=n(1),i=n(165)(!1);r(r.S,"Object",{values:function(e){return i(e)}})},function(e,t,n){var r=n(1),i=n(165)(!0);r(r.S,"Object",{entries:function(e){return i(e)}})},function(e,t,n){"use strict";var r=n(1),i=n(17),o=n(19),a=n(15);n(12)&&r(r.P+n(89),"Object",{__defineGetter__:function(e,t){a.f(i(this),e,{get:o(t),enumerable:!0,configurable:!0})}})},function(e,t,n){"use strict";var r=n(1),i=n(17),o=n(19),a=n(15);n(12)&&r(r.P+n(89),"Object",{__defineSetter__:function(e,t){a.f(i(this),e,{set:o(t),enumerable:!0,configurable:!0})}})},function(e,t,n){"use strict";var r=n(1),i=n(17),o=n(38),a=n(27),s=n(26).f;n(12)&&r(r.P+n(89),"Object",{__lookupGetter__:function(e){var t,n=i(this),r=o(e,!0);do{if(t=s(n,r))return t.get}while(n=a(n))}})},function(e,t,n){"use strict";var r=n(1),i=n(17),o=n(38),a=n(27),s=n(26).f;n(12)&&r(r.P+n(89),"Object",{__lookupSetter__:function(e){var t,n=i(this),r=o(e,!0);do{if(t=s(n,r))return t.set}while(n=a(n))}})},function(e,t,n){var r=n(1);r(r.P+r.R,"Map",{toJSON:n(166)("Map")})},function(e,t,n){var r=n(1);r(r.P+r.R,"Set",{toJSON:n(166)("Set")})},function(e,t,n){n(90)("Map")},function(e,t,n){n(90)("Set")},function(e,t,n){n(90)("WeakMap")},function(e,t,n){n(90)("WeakSet")},function(e,t,n){n(91)("Map")},function(e,t,n){n(91)("Set")},function(e,t,n){n(91)("WeakMap")},function(e,t,n){n(91)("WeakSet")},function(e,t,n){var r=n(1);r(r.G,{global:n(6)})},function(e,t,n){var r=n(1);r(r.S,"System",{global:n(6)})},function(e,t,n){var r=n(1),i=n(32);r(r.S,"Error",{isError:function(e){return"Error"===i(e)}})},function(e,t,n){var r=n(1);r(r.S,"Math",{clamp:function(e,t,n){return Math.min(n,Math.max(t,e))}})},function(e,t,n){var r=n(1);r(r.S,"Math",{DEG_PER_RAD:Math.PI/180})},function(e,t,n){var r=n(1),i=180/Math.PI;r(r.S,"Math",{degrees:function(e){return e*i}})},function(e,t,n){var r=n(1),o=n(168),a=n(149);r(r.S,"Math",{fscale:function(e,t,n,r,i){return a(o(e,t,n,r,i))}})},function(e,t,n){var r=n(1);r(r.S,"Math",{iaddh:function(e,t,n,r){var i=e>>>0,o=n>>>0;return(t>>>0)+(r>>>0)+((i&o|(i|o)&~(i+o>>>0))>>>31)|0}})},function(e,t,n){var r=n(1);r(r.S,"Math",{isubh:function(e,t,n,r){var i=e>>>0,o=n>>>0;return(t>>>0)-(r>>>0)-((~i&o|~(i^o)&i-o>>>0)>>>31)|0}})},function(e,t,n){var r=n(1);r(r.S,"Math",{imulh:function(e,t){var n=+e,r=+t,i=65535&n,o=65535&r,a=n>>16,s=r>>16,l=(a*o>>>0)+(i*o>>>16);return a*s+(l>>16)+((i*s>>>0)+(65535&l)>>16)}})},function(e,t,n){var r=n(1);r(r.S,"Math",{RAD_PER_DEG:180/Math.PI})},function(e,t,n){var r=n(1),i=Math.PI/180;r(r.S,"Math",{radians:function(e){return e*i}})},function(e,t,n){var r=n(1);r(r.S,"Math",{scale:n(168)})},function(e,t,n){var r=n(1);r(r.S,"Math",{umulh:function(e,t){var n=+e,r=+t,i=65535&n,o=65535&r,a=n>>>16,s=r>>>16,l=(a*o>>>0)+(i*o>>>16);return a*s+(l>>>16)+((i*s>>>0)+(65535&l)>>>16)}})},function(e,t,n){var r=n(1);r(r.S,"Math",{signbit:function(e){return(e=+e)!=e?e:0==e?1/e==1/0:0=n.length)return this._t=void 0,y(1)}while(!x(t,e=n[this._i++]));return y(0,"keys"==r?e:"values"==r?t[e]:[e,t[e]])}),O.prototype=null,o(o.G+o.F,{Dict:O}),o(o.S,"Dict",{keys:i("keys"),values:i("values"),entries:i("entries"),forEach:r(0),map:r(1),filter:r(2),some:r(3),every:r(4),find:r(5),findKey:S,mapPairs:r(7),reduce:function(e,t,n){h(t);var r,i,o=w(e),a=c(o),s=a.length,l=0;if(arguments.length<3){if(!s)throw TypeError("Reduce of empty object with no initial value");r=o[a[l++]]}else r=Object(n);for(;l"']/g,{"&":"&","<":"<",">":">",'"':""","'":"'"});r(r.P+r.F,"String",{escapeHTML:function(){return i(this)}})},function(e,t,n){"use strict";var r=n(1),i=n(124)(/&(?:amp|lt|gt|quot|apos);/g,{"&":"&","<":"<",">":">",""":'"',"'":"'"});r(r.P+r.F,"String",{unescapeHTML:function(){return i(this)}})},function(e,t,n){"use strict";var c=n(173),r="function"==typeof Symbol&&Symbol.for,f=r?Symbol.for("react.element"):60103,u=r?Symbol.for("react.portal"):60106,i=r?Symbol.for("react.fragment"):60107,o=r?Symbol.for("react.strict_mode"):60108,a=r?Symbol.for("react.profiler"):60114,s=r?Symbol.for("react.provider"):60109,l=r?Symbol.for("react.context"):60110,d=r?Symbol.for("react.forward_ref"):60112,p=r?Symbol.for("react.suspense"):60113,h=r?Symbol.for("react.memo"):60115,v=r?Symbol.for("react.lazy"):60116,m="function"==typeof Symbol&&Symbol.iterator;function g(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n