ATLAS-3386 : Atlas documentation website with new template
Signed-off-by: nixonrodrigues <nixon@apache.org>
This commit is contained in:
kevalbhatt
nixonrodrigues
parent
05130e83a1
commit
f31db4c397
|
|
@ -0,0 +1,24 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
.DS_Store
|
||||
.bower-*/
|
||||
.idea/
|
||||
node_modules/
|
||||
.docz
|
||||
target/
|
||||
*.log
|
||||
*.tgz
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
export default {
|
||||
'~docz-theme-component': '../node_modules/theme/dist/components/ui/index.ts'
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
export default [
|
||||
"Introduction",
|
||||
{
|
||||
name: "Documentation",
|
||||
menu: [
|
||||
{
|
||||
name: "Features",
|
||||
menu: [
|
||||
"High Level Architecture",
|
||||
"Type System",
|
||||
"Glossary",
|
||||
"Classification Propagation",
|
||||
"Notifications",
|
||||
"High Availability"
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Search",
|
||||
menu: [
|
||||
"Basic Search",
|
||||
"Advance Search"
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Hooks",
|
||||
menu: [
|
||||
"HBase",
|
||||
"Hive",
|
||||
"Sqoop",
|
||||
"Storm",
|
||||
"Kafka",
|
||||
"Falcon"
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Import/Export",
|
||||
menu: [
|
||||
"Import API",
|
||||
"Import API Options",
|
||||
"Import Entity Transforms",
|
||||
"Import Export API",
|
||||
"Export API",
|
||||
"Export HDFS API",
|
||||
"Export Import Audits",
|
||||
"Incremental Export"
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Security",
|
||||
menu: [
|
||||
"Security Details",
|
||||
"Authentication",
|
||||
"Authorization Model",
|
||||
"Atlas Simple Authorizer",
|
||||
"Atlas Ranger Authorizer"
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Setup",
|
||||
menu: [
|
||||
"Quick Start",
|
||||
"Configuration",
|
||||
"Build Instruction",
|
||||
"Installation Instruction"
|
||||
]
|
||||
},
|
||||
"REST API",
|
||||
{
|
||||
name: "Tools",
|
||||
menu: [
|
||||
"Atlas Repair Index"
|
||||
]
|
||||
},
|
||||
{
|
||||
name: "Misc",
|
||||
menu: [
|
||||
"Atlas Server",
|
||||
"Replicated Attributes",
|
||||
"Soft Reference"
|
||||
]
|
||||
}
|
||||
]
|
||||
}, {
|
||||
name: "Downloads",
|
||||
menu: [
|
||||
"Download",
|
||||
{
|
||||
name: "Whats New",
|
||||
menu: ["WhatsNew-2.0", "WhatsNew-1.0"]
|
||||
},
|
||||
"Migration"
|
||||
]
|
||||
} ,"For Developers",
|
||||
{
|
||||
name: "Project Info", menu: [
|
||||
"Project Information",
|
||||
"Mailing Lists",
|
||||
"Team List",
|
||||
"Issue Tracking",
|
||||
"License",
|
||||
"Source Repository"
|
||||
]
|
||||
}, "ASF" ];
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
export default [
|
||||
{ id: 1, href: "/Doc-test/", title: "Latest", label: "Latest" },
|
||||
{ id: 2, href: "/2.0.0/index.html", title: "2.0.0", label: "2.0.0" },
|
||||
{ id: 4, href: "/1.1.0/index.html", title: "1.1.0", label: "1.1.0" },
|
||||
{ id: 5, href: "/1.0.0/index.html", title: "1.0.0", label: "1.0.0" },
|
||||
{ id: 7, href: "/0.8.3/index.html", title: "0.8.3", label: "0.8.3" },
|
||||
{ id: 8, href: "/0.8.2/index.html", title: "0.8.2", label: "0.8.2" },
|
||||
{ id: 9, href: "/0.8.1/index.html", title: "0.8.1", label: "0.8.1" },
|
||||
{
|
||||
id: 10,
|
||||
href: "/0.8.0-incubating/index.html",
|
||||
title: "0.8-incubating",
|
||||
label: "0.8-incubating"
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
href: "/0.7.1-incubating/index.html",
|
||||
title: "0.7.1-incubating",
|
||||
label: "0.7.1-incubating"
|
||||
},
|
||||
{
|
||||
id: 12,
|
||||
href: "/0.7.0-incubating/index.html",
|
||||
title: "0.7-incubating",
|
||||
label: "0.7-incubating"
|
||||
},
|
||||
{
|
||||
id: 13,
|
||||
href: "/0.6.0-incubating/index.html",
|
||||
title: "0.6-incubating",
|
||||
label: "0.6-incubating"
|
||||
},
|
||||
{
|
||||
id: 14,
|
||||
href: "/0.5.0-incubating/index.html",
|
||||
title: "0.5-incubating",
|
||||
label: "0.5-incubating"
|
||||
}
|
||||
];
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
import webpack from 'webpack';
|
||||
export declare const build: (config: webpack.Configuration, dist: string, publicDir: string) => Promise<void>;
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
import { Configuration } from 'webpack';
|
||||
import { ServerHooks as Hooks } from '../lib/Bundler';
|
||||
import { Config as Args, Env } from '../config/argv';
|
||||
export declare const createConfig: (args: Args, env: Env) => (hooks: Hooks) => Promise<Configuration>;
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import { Config as Args } from '../config/argv';
|
||||
import { ServerHooks } from '../lib/Bundler';
|
||||
export declare const devServerConfig: (hooks: ServerHooks, args: Args) => {
|
||||
publicPath: string;
|
||||
compress: boolean;
|
||||
logLevel: string;
|
||||
clientLogLevel: string;
|
||||
contentBase: string[];
|
||||
watchContentBase: boolean;
|
||||
hot: boolean;
|
||||
quiet: boolean;
|
||||
open: boolean;
|
||||
watchOptions: {
|
||||
ignored: any;
|
||||
};
|
||||
overlay: boolean;
|
||||
host: string;
|
||||
port: number;
|
||||
historyApiFallback: {
|
||||
disableDotRule: boolean;
|
||||
};
|
||||
disableHostCheck: boolean;
|
||||
before(app: any, server: any): void;
|
||||
after(app: any): void;
|
||||
};
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
import { Configuration as CFG } from 'webpack';
|
||||
import { Bundler } from '../lib/Bundler';
|
||||
import { Config as Args, Env } from '../config/argv';
|
||||
export declare const bundler: (args: Args, env: Env) => Bundler<CFG>;
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import Config from 'webpack-chain';
|
||||
import { Config as Args } from '../config/argv';
|
||||
import { BabelRC } from '../config/babel';
|
||||
export declare const sourceMaps: (config: Config, args: Args) => void;
|
||||
export interface AddScriptLoaderOpts {
|
||||
threadLoader?: boolean;
|
||||
rule: Config.Rule;
|
||||
babelrc: BabelRC;
|
||||
args: Args;
|
||||
}
|
||||
export declare const js: (config: Config, args: Args, babelrc: BabelRC) => void;
|
||||
export declare const ts: (config: Config, args: Args, babelrc: BabelRC) => void;
|
||||
export declare const mdx: (config: Config, args: Args, babelrc: BabelRC) => void;
|
||||
export declare const images: (config: Config) => void;
|
||||
export declare const svg: (config: Config) => void;
|
||||
export declare const media: (config: Config) => void;
|
||||
export declare const fonts: (config: Config) => void;
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
import Config from 'webpack-chain';
|
||||
import { Config as Args } from '../config/argv';
|
||||
export declare const minifier: (config: Config, args: Args) => void;
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import Config from 'webpack-chain';
|
||||
import { Config as Args, Env } from '../config/argv';
|
||||
export declare const assets: (config: Config, args: Args, env: Env) => void;
|
||||
export declare const analyzer: (config: Config) => void;
|
||||
export declare const injections: (config: Config, args: Args, env: Env) => void;
|
||||
export declare const ignore: (config: Config) => void;
|
||||
export declare const hot: (config: Config) => void;
|
||||
export declare const html: (config: Config, args: Args, env: Env) => Promise<void>;
|
||||
export declare const webpackBar: (config: Config, args: Args) => void;
|
||||
export declare const watchNodeModulesPlugin: (config: Config) => void;
|
||||
export declare const notFoundPlugin: (config: Config) => void;
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
import webpack from 'webpack';
|
||||
import { Config as Args } from '../config/argv';
|
||||
import { ServerHooks as Hooks } from '../lib/Bundler';
|
||||
export declare const server: (args: Args) => (config: webpack.Configuration, hooks: Hooks) => Promise<{
|
||||
start: () => Promise<any>;
|
||||
}>;
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export declare const cli: () => {
|
||||
[x: string]: unknown;
|
||||
_: string[];
|
||||
$0: string;
|
||||
};
|
||||
|
|
@ -0,0 +1 @@
|
|||
export declare const build: (args: any) => Promise<void>;
|
||||
|
|
@ -0,0 +1 @@
|
|||
export declare const dev: (args: any) => Promise<void>;
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
export { dev } from './dev';
|
||||
export { build } from './build';
|
||||
export { serve } from './serve';
|
||||
|
|
@ -0,0 +1 @@
|
|||
export declare const serve: (args: any) => Promise<void>;
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
import { Argv as Yargs } from 'yargs';
|
||||
import { Plugin } from '../lib/Plugin';
|
||||
import { BabelRC } from '../config/babel';
|
||||
export declare type Env = 'production' | 'development';
|
||||
export declare type ThemeConfig = Record<string, any>;
|
||||
export interface DocgenConfig {
|
||||
handlers?: any[];
|
||||
resolver?: (ast: any, recast: any) => any;
|
||||
propFilter?: (prop: any) => boolean;
|
||||
searchPath: string;
|
||||
}
|
||||
export interface Menu {
|
||||
name: string;
|
||||
route?: string;
|
||||
href?: string;
|
||||
menu?: Menu[];
|
||||
}
|
||||
export interface HtmlContext {
|
||||
lang: string;
|
||||
favicon?: string;
|
||||
head?: {
|
||||
meta: any[];
|
||||
links: any[];
|
||||
raw: string;
|
||||
scripts: any[];
|
||||
};
|
||||
body?: {
|
||||
raw: string;
|
||||
scripts: any[];
|
||||
};
|
||||
}
|
||||
export interface Argv {
|
||||
base: string;
|
||||
src: string;
|
||||
files: string | string[];
|
||||
ignore: string[];
|
||||
watchIgnore: string;
|
||||
public: string;
|
||||
dest: string;
|
||||
editBranch: string;
|
||||
config: string;
|
||||
debug: boolean;
|
||||
clearConsole: boolean;
|
||||
typescript: boolean;
|
||||
propsParser: boolean;
|
||||
host: string;
|
||||
port: number;
|
||||
websocketPort: number;
|
||||
websocketHost: string;
|
||||
native: boolean;
|
||||
codeSandbox: boolean;
|
||||
sourcemaps: boolean;
|
||||
notUseSpecifiers: boolean;
|
||||
title: string;
|
||||
description: string;
|
||||
theme: string;
|
||||
wrapper?: string;
|
||||
indexHtml?: string;
|
||||
/** slugify separator */
|
||||
separator: string;
|
||||
}
|
||||
export interface Config extends Argv {
|
||||
paths: Record<string, any>;
|
||||
plugins: Plugin[];
|
||||
mdPlugins: any[];
|
||||
hastPlugins: any[];
|
||||
menu: Menu[];
|
||||
htmlContext: HtmlContext;
|
||||
themeConfig: ThemeConfig;
|
||||
docgenConfig: DocgenConfig;
|
||||
filterComponents: (files: string[]) => string[];
|
||||
modifyBundlerConfig<C>(config: C, dev: boolean, args: Config): C;
|
||||
modifyBabelRc(babelrc: BabelRC, args: Config): BabelRC;
|
||||
onCreateWebpackChain<C>(c: C, dev: boolean, args: Config): void;
|
||||
}
|
||||
export declare const setArgs: (yargs: Yargs<{}>) => Yargs<{
|
||||
base: any;
|
||||
} & {
|
||||
source: any;
|
||||
} & {
|
||||
files: any;
|
||||
} & {
|
||||
ignore: any;
|
||||
} & {
|
||||
public: any;
|
||||
} & {
|
||||
dest: any;
|
||||
} & {
|
||||
editBranch: any;
|
||||
} & {
|
||||
config: any;
|
||||
} & {
|
||||
title: any;
|
||||
} & {
|
||||
description: any;
|
||||
} & {
|
||||
theme: any;
|
||||
} & {
|
||||
typescript: any;
|
||||
} & {
|
||||
propsParser: any;
|
||||
} & {
|
||||
wrapper: any;
|
||||
} & {
|
||||
indexHtml: any;
|
||||
} & {
|
||||
debug: any;
|
||||
} & {
|
||||
clearConsole: any;
|
||||
} & {
|
||||
host: any;
|
||||
} & {
|
||||
port: any;
|
||||
} & {
|
||||
websocketHost: any;
|
||||
} & {
|
||||
websocketPort: any;
|
||||
} & {
|
||||
native: any;
|
||||
} & {
|
||||
codeSandbox: any;
|
||||
} & {
|
||||
sourcemaps: any;
|
||||
} & {
|
||||
separator: any;
|
||||
} & {
|
||||
open: boolean;
|
||||
}>;
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { Config, Env } from '../config/argv';
|
||||
export interface BabelRC {
|
||||
presets: any[];
|
||||
plugins: any[];
|
||||
cacheDirectory?: boolean;
|
||||
babelrc?: boolean;
|
||||
}
|
||||
export declare const getBabelConfig: (args: Config, env: Env, typescript?: boolean | undefined) => Promise<BabelRC>;
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import { Arguments } from 'yargs';
|
||||
import { BabelRC } from '../config/babel';
|
||||
import { Config, Argv } from '../config/argv';
|
||||
export declare const doczRcBaseConfig: {
|
||||
htmlContext: {
|
||||
lang: string;
|
||||
favicon: string;
|
||||
};
|
||||
themeConfig: {};
|
||||
docgenConfig: {};
|
||||
filterComponents: (files: string[]) => string[];
|
||||
modifyBundlerConfig: (config: any) => any;
|
||||
modifyBabelRc: (babelrc: BabelRC) => BabelRC;
|
||||
onCreateWebpackChain: () => null;
|
||||
menu: never[];
|
||||
plugins: never[];
|
||||
mdPlugins: never[];
|
||||
hastPlugins: never[];
|
||||
ignore: string[];
|
||||
};
|
||||
export declare const getBaseConfig: (argv: Arguments<Argv>, custom?: Partial<Config> | undefined) => Config;
|
||||
export declare const parseConfig: (argv: Arguments<Argv>, custom?: Partial<Config> | undefined) => Promise<Config>;
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
export declare const setEnv: (env: string) => void;
|
||||
export interface RT {
|
||||
[key: string]: any;
|
||||
}
|
||||
export declare const getClientEnvironment: (publicUrl: string) => {
|
||||
raw: RT;
|
||||
stringified: {
|
||||
'process.env': {};
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
export declare const ensureSlash: (filepath: any, needsSlash: boolean) => any;
|
||||
export declare const root: string;
|
||||
export declare const resolveApp: (to: string) => string;
|
||||
export declare const resolveOwn: (to: string) => string;
|
||||
export interface Paths {
|
||||
root: string;
|
||||
templates: string;
|
||||
packageJson: string;
|
||||
servedPath: (base: string) => string;
|
||||
docz: string;
|
||||
app: string;
|
||||
cache: string;
|
||||
appPublic: string;
|
||||
appNodeModules: string;
|
||||
appPackageJson: string;
|
||||
appYarnLock: string;
|
||||
ownNodeModules: string;
|
||||
getDist: (dest: string) => string;
|
||||
distPublic: (dest: string) => string;
|
||||
importsJs: string;
|
||||
rootJs: string;
|
||||
indexJs: string;
|
||||
indexHtml: string;
|
||||
db: string;
|
||||
}
|
||||
export declare const templates: string;
|
||||
export declare const packageJson: string;
|
||||
export declare const servedPath: (base: string) => any;
|
||||
export declare const docz: string;
|
||||
export declare const app: string;
|
||||
export declare const cache: string;
|
||||
export declare const appPublic: string;
|
||||
export declare const appNodeModules: string;
|
||||
export declare const appPackageJson: string;
|
||||
export declare const appYarnLock: string;
|
||||
export declare const ownNodeModules: string;
|
||||
export declare const getDist: (dest: string) => string;
|
||||
export declare const distPublic: (dest: string) => string;
|
||||
export declare const importsJs: string;
|
||||
export declare const rootJs: string;
|
||||
export declare const indexJs: string;
|
||||
export declare const indexHtml: string;
|
||||
export declare const db: string;
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
/** cli exports */
|
||||
export { cli } from './cli';
|
||||
/** config exports */
|
||||
export { parseConfig, getBaseConfig } from './config/docz';
|
||||
export { Config, setArgs } from './config/argv';
|
||||
export { BabelRC } from './config/babel';
|
||||
/** states */
|
||||
import * as states from './states';
|
||||
export { states };
|
||||
/** lib exports */
|
||||
export { Plugin, createPlugin } from './lib/Plugin';
|
||||
export { DataServer } from './lib/DataServer';
|
||||
export { Entries } from './lib/Entries';
|
||||
export { Entry } from './lib/Entry';
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,35 @@
|
|||
/// <reference types="node" />
|
||||
import * as http from 'http';
|
||||
import { Config as Args, Env } from '../config/argv';
|
||||
export interface ServerHooks {
|
||||
onCreateWebpackChain<C>(config: C, dev: boolean, args: Args): void;
|
||||
onPreCreateApp<A>(app: A): void;
|
||||
onCreateApp<A>(app: A): void;
|
||||
onServerListening<S>(server: S): void;
|
||||
}
|
||||
export interface BundlerServer {
|
||||
start(): Promise<http.Server>;
|
||||
}
|
||||
export declare type ConfigFn<C> = (hooks: ServerHooks) => Promise<C>;
|
||||
export declare type BuildFn<C> = (config: C, dist: string, publicDir: string) => void;
|
||||
export declare type ServerFn<C> = (config: C, hooks: ServerHooks) => BundlerServer | Promise<BundlerServer>;
|
||||
export interface BundlerConstructor<Config> {
|
||||
args: Args;
|
||||
config: ConfigFn<Config>;
|
||||
server: ServerFn<Config>;
|
||||
build: BuildFn<Config>;
|
||||
}
|
||||
export interface ConfigObj {
|
||||
[key: string]: any;
|
||||
}
|
||||
export declare class Bundler<C = ConfigObj> {
|
||||
private readonly args;
|
||||
private config;
|
||||
private server;
|
||||
private builder;
|
||||
private hooks;
|
||||
constructor(params: BundlerConstructor<C>);
|
||||
mountConfig(env: Env): Promise<C>;
|
||||
createApp(config: C): Promise<BundlerServer>;
|
||||
build(config: C): Promise<void>;
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
export interface Params {
|
||||
getState: () => Record<string, any>;
|
||||
setState: (key: string, val: any) => void;
|
||||
}
|
||||
export interface State {
|
||||
id: string;
|
||||
start: (params: Params) => Promise<void>;
|
||||
close: () => void;
|
||||
}
|
||||
export interface Action {
|
||||
type: string;
|
||||
payload: any;
|
||||
}
|
||||
export declare type Listener = (action: Action) => void;
|
||||
export declare class DataServer {
|
||||
private states;
|
||||
private state;
|
||||
private listeners;
|
||||
constructor();
|
||||
register(states: State[]): DataServer;
|
||||
start(): Promise<void>;
|
||||
close(): void;
|
||||
onStateChange(listener: Listener): () => void;
|
||||
getState(): Map<string, any>;
|
||||
private setState;
|
||||
private writeDbFile;
|
||||
private mapToObject;
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import { EntryObj } from './Entry';
|
||||
import { Config } from '../config/argv';
|
||||
export declare const fromTemplates: (file: string) => string;
|
||||
export declare type EntryMap = Record<string, EntryObj>;
|
||||
export declare class Entries {
|
||||
static writeApp(config: Config, dev: boolean): Promise<void>;
|
||||
static writeImports(map: EntryMap): Promise<void>;
|
||||
all: Map<string, EntryObj>;
|
||||
get: () => Promise<EntryMap>;
|
||||
repoEditUrl: string | null;
|
||||
constructor(config: Config);
|
||||
private getMap;
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { Heading } from 'docz-utils/lib/mdast';
|
||||
import { Config } from '../config/argv';
|
||||
export interface EntryObj {
|
||||
id: string;
|
||||
filepath: string;
|
||||
link: string | null;
|
||||
slug: string;
|
||||
name: string;
|
||||
route: string;
|
||||
menu: string | null;
|
||||
headings: Heading[];
|
||||
[key: string]: any;
|
||||
}
|
||||
export declare class Entry {
|
||||
readonly [key: string]: any;
|
||||
id: string;
|
||||
filepath: string;
|
||||
link: string | null;
|
||||
slug: string;
|
||||
route: string;
|
||||
name: string;
|
||||
menu: string | null;
|
||||
headings: Heading[];
|
||||
settings: {
|
||||
[key: string]: any;
|
||||
};
|
||||
constructor(ast: any, file: string, src: string, config: Config);
|
||||
setLink(url: string): void;
|
||||
private getFilepath;
|
||||
private getName;
|
||||
private slugify;
|
||||
private getRoute;
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
import WebpackChainConfig from 'webpack-chain';
|
||||
import { Config } from '../config/argv';
|
||||
import { BabelRC } from '../config/babel';
|
||||
export declare type SetConfig = (config: Config) => Config | Promise<Config>;
|
||||
export declare type ModifyBundlerConfig<C = any> = (config: C, dev: boolean, args: Config) => C;
|
||||
export declare type ModifyBabelRC = (babelrc: BabelRC, args: Config) => BabelRC;
|
||||
export declare type ModifyFiles = (files: string[], args: Config) => string[];
|
||||
export declare type OnCreateWebpackChain = (config: WebpackChainConfig, dev: boolean, args: Config) => void;
|
||||
export declare type onPreCreateApp = <A>(app: A) => void;
|
||||
export declare type onCreateApp = <A>(app: A) => void;
|
||||
export declare type OnServerListening = <S>(server: S) => void;
|
||||
export declare type OnPreBuild = (args: Config) => void;
|
||||
export declare type OnPostBuild = (args: Config) => void;
|
||||
export declare type OnPreRender = () => void;
|
||||
export declare type OnPostRender = () => void;
|
||||
export interface PluginFactory {
|
||||
setConfig?: SetConfig;
|
||||
modifyBundlerConfig?: ModifyBundlerConfig;
|
||||
modifyBabelRc?: ModifyBabelRC;
|
||||
modifyFiles?: ModifyFiles;
|
||||
onCreateWebpackChain?: OnCreateWebpackChain;
|
||||
onPreCreateApp?: onPreCreateApp;
|
||||
onCreateApp?: onCreateApp;
|
||||
onServerListening?: OnServerListening;
|
||||
onPreBuild?: OnPreBuild;
|
||||
onPostBuild?: OnPostBuild;
|
||||
onPreRender?: OnPreRender;
|
||||
onPostRender?: OnPostRender;
|
||||
}
|
||||
export declare class Plugin<C = any> implements PluginFactory {
|
||||
static runPluginsMethod(plugins: Plugin[] | undefined): (method: keyof Plugin, ...args: any[]) => void;
|
||||
static propsOfPlugins(plugins: Plugin[]): (prop: keyof Plugin) => any[];
|
||||
static reduceFromPlugins<C>(plugins: Plugin[] | undefined): (method: keyof Plugin, initial: C, ...args: any[]) => C;
|
||||
static reduceFromPluginsAsync<C>(plugins: Plugin[] | undefined): (method: keyof Plugin, initial: C, ...args: any[]) => Promise<C>;
|
||||
readonly setConfig?: SetConfig;
|
||||
readonly modifyBundlerConfig?: ModifyBundlerConfig<C>;
|
||||
readonly modifyBabelRc?: ModifyBabelRC;
|
||||
readonly modifyFiles?: ModifyFiles;
|
||||
readonly onCreateWebpackChain?: OnCreateWebpackChain;
|
||||
readonly onPreCreateApp?: onPreCreateApp;
|
||||
readonly onCreateApp?: onCreateApp;
|
||||
readonly onServerListening?: OnServerListening;
|
||||
readonly onPreBuild?: OnPreBuild;
|
||||
readonly onPostBuild?: OnPostBuild;
|
||||
readonly onPreRender?: OnPreRender;
|
||||
readonly onPostRender?: OnPostRender;
|
||||
constructor(p: PluginFactory);
|
||||
}
|
||||
export declare function createPlugin<C = any>(factory: PluginFactory): Plugin<C>;
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import WS from 'ws';
|
||||
export declare type Send = (type: string, payload: any) => void;
|
||||
export declare type On = (type: string) => Promise<any>;
|
||||
export declare class Socket {
|
||||
private client?;
|
||||
constructor(server?: any, host?: string, port?: number);
|
||||
onConnection(listener: (socket: WS, emit: Send) => () => void): void;
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
import { State } from '../lib/DataServer';
|
||||
import { Config } from '../config/argv';
|
||||
export declare const state: (config: Config, dev?: boolean | undefined) => State;
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
import { State } from '../lib/DataServer';
|
||||
import { Entries } from '../lib/Entries';
|
||||
import { Config } from '../config/argv';
|
||||
export declare const state: (entries: Entries, config: Config, dev?: boolean | undefined) => State;
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
export { state as entries } from './entries';
|
||||
export { state as config } from './config';
|
||||
export { state as props } from './props';
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
import { State } from '../lib/DataServer';
|
||||
import { Config } from '../config/argv';
|
||||
export declare const state: (config: Config, dev?: boolean | undefined) => State;
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
export const imports = {
|
||||
<% entries.forEach(function(entry) { %>'<%- entry.filepath %>': () =>
|
||||
import(/* webpackPrefetch: true, webpackChunkName: "<%- entry.slug %>" */ '<%- entry.filepath %>'),<% }) %>
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="{{ lang }}">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="description" content="{{ description }}">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||||
<title>{{ title }}</title>
|
||||
{{ head }}
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
{{ footer }}
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import React from 'react'
|
||||
import ReactDOM from 'react-dom'
|
||||
import Root from './root'
|
||||
|
||||
const _onPreRenders = [<% if (onPreRenders) {%><%- onPreRenders %><%}%>]
|
||||
const _onPostRenders = [<% if (onPostRenders) {%><%- onPostRenders %><%}%>]
|
||||
|
||||
const onPreRender = () => _onPreRenders.forEach(f => f && f())
|
||||
const onPostRender = () => _onPostRenders.forEach(f => f && f())
|
||||
|
||||
const root = document.querySelector('#root')
|
||||
const render = (Component = Root) => {
|
||||
onPreRender()
|
||||
ReactDOM.render(<Component />, root, onPostRender)
|
||||
}
|
||||
|
||||
render(Root)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import React from 'react'
|
||||
import { Link, Router, Routes<% if (!isProd) {%>, useDataServer<%}%> } from '../../docz-lib/docz/dist'
|
||||
<% if (!isProd) {%>import { hot } from 'react-hot-loader'<%}%>
|
||||
import Theme from '<%- theme %>'
|
||||
|
||||
import { imports } from './imports'
|
||||
import database from './db.json'
|
||||
<% if (wrapper) {%>import Wrapper from '<%- wrapper %>'<%}%>
|
||||
|
||||
const Root = () => {
|
||||
<% if (!isProd && websocketUrl) {%>useDataServer('<%- websocketUrl %>')<%}%>
|
||||
return (
|
||||
<Theme
|
||||
<% if (wrapper) {%>wrapper={Wrapper}<%}%>
|
||||
linkComponent={Link}
|
||||
db={database}
|
||||
>
|
||||
<Routes imports={imports} />
|
||||
</Theme>
|
||||
)
|
||||
}
|
||||
|
||||
<% if (!isProd) {%>export default hot(module)(Root)<%}%>
|
||||
<% if (isProd) {%>export default Root<%}%>
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
import { Config } from '../../config/argv';
|
||||
export declare const docgen: (files: string[], config: Config) => Promise<any>;
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
import { Config } from '../../config/argv';
|
||||
export declare const jsParser: (files: string[], config: Config) => ({
|
||||
key: string;
|
||||
value: any;
|
||||
} | null)[];
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
import { Config } from '../../config/argv';
|
||||
export interface TSFile {
|
||||
text?: string;
|
||||
version: number;
|
||||
}
|
||||
export declare const tsParser: (files: string[], config: Config, tsconfig?: string | undefined) => any;
|
||||
|
|
@ -0,0 +1 @@
|
|||
export declare const onSignal: (cb: () => any) => void;
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/**
|
||||
* Copyright (c) 2015-present, Facebook, Inc.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
/**
|
||||
* Reads the BROWSER environment variable and decides what to do with it. Returns
|
||||
* true if it opened a browser or ran a node.js script, otherwise false.
|
||||
*/
|
||||
export declare function openBrowser(url: any): boolean;
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import { Config } from '../config/argv';
|
||||
export declare type tagsTemplate = (type: string) => string;
|
||||
export declare const htmlTemplate: (indexHtml: string | undefined) => Promise<(data: any) => string>;
|
||||
interface ParseHtmlParams {
|
||||
config: Config;
|
||||
ctx: Record<string, any>;
|
||||
dev: boolean;
|
||||
template: (props: Record<string, any>) => string;
|
||||
}
|
||||
export declare const parseHtml: ({ config, ctx, dev, template }: ParseHtmlParams) => any;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
export declare const parseRepo: () => any;
|
||||
export declare const getRepoUrl: () => any;
|
||||
export declare const getRepoEditUrl: (src: string, branch: string) => string | null;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#!/usr/bin/env nodejs
|
||||
require('../../docz-core/dist/index.js').cli()
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import { createElement } from 'react';
|
||||
import 'lodash/fp/get';
|
||||
import { b as useComponents } from './chunk.esm.js';
|
||||
import 'lodash/fp/omit';
|
||||
import 'fast-deep-equal';
|
||||
import 'lodash/fp/merge';
|
||||
import 'array-sort';
|
||||
import 'lodash/fp/unionBy';
|
||||
import 'lodash/fp/flattenDepth';
|
||||
import 'lodash/fp/pipe';
|
||||
import 'ulid';
|
||||
import 'match-sorter';
|
||||
import 'lodash/fp/throttle';
|
||||
|
||||
const Playground = ({
|
||||
className,
|
||||
style,
|
||||
wrapper: Wrapper,
|
||||
children,
|
||||
__scope,
|
||||
__position,
|
||||
__code,
|
||||
__codesandbox
|
||||
}) => {
|
||||
const components = useComponents();
|
||||
if (!components || !components.playground) return null;
|
||||
const props = {
|
||||
className,
|
||||
style,
|
||||
components
|
||||
};
|
||||
return createElement(components.playground, Object.assign({}, props, {
|
||||
component: children,
|
||||
wrapper: Wrapper,
|
||||
scope: __scope,
|
||||
position: __position,
|
||||
code: __code,
|
||||
codesandbox: __codesandbox
|
||||
}));
|
||||
};
|
||||
|
||||
export default Playground;
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
'use strict';
|
||||
|
||||
var React = require('react');
|
||||
require('lodash/fp/get');
|
||||
var __chunk_1 = require('./chunk.js');
|
||||
require('lodash/fp/omit');
|
||||
require('fast-deep-equal');
|
||||
require('lodash/fp/merge');
|
||||
require('array-sort');
|
||||
require('lodash/fp/unionBy');
|
||||
require('lodash/fp/flattenDepth');
|
||||
require('lodash/fp/pipe');
|
||||
require('ulid');
|
||||
require('match-sorter');
|
||||
require('lodash/fp/throttle');
|
||||
|
||||
const Playground = ({
|
||||
className,
|
||||
style,
|
||||
wrapper: Wrapper,
|
||||
children,
|
||||
__scope,
|
||||
__position,
|
||||
__code,
|
||||
__codesandbox
|
||||
}) => {
|
||||
const components = __chunk_1.useComponents();
|
||||
if (!components || !components.playground) return null;
|
||||
const props = {
|
||||
className,
|
||||
style,
|
||||
components
|
||||
};
|
||||
return React.createElement(components.playground, Object.assign({}, props, {
|
||||
component: children,
|
||||
wrapper: Wrapper,
|
||||
scope: __scope,
|
||||
position: __position,
|
||||
code: __code,
|
||||
codesandbox: __codesandbox
|
||||
}));
|
||||
};
|
||||
|
||||
exports.default = Playground;
|
||||
|
|
@ -0,0 +1,311 @@
|
|||
import { createContext, createElement, useContext, Fragment, Component, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import _get from 'lodash/fp/get';
|
||||
import _omit from 'lodash/fp/omit';
|
||||
import equal from 'fast-deep-equal';
|
||||
import _merge from 'lodash/fp/merge';
|
||||
import sort from 'array-sort';
|
||||
import _unionBy from 'lodash/fp/unionBy';
|
||||
import _flattenDepth from 'lodash/fp/flattenDepth';
|
||||
import _pipe from 'lodash/fp/pipe';
|
||||
import { ulid } from 'ulid';
|
||||
import match from 'match-sorter';
|
||||
import _throttle from 'lodash/fp/throttle';
|
||||
|
||||
const DefaultNotFound = () => createElement(Fragment, null, "Not found");
|
||||
|
||||
const DefaultLoading = () => createElement(Fragment, null, "Loading");
|
||||
|
||||
const DefaultPage = ({
|
||||
children
|
||||
}) => createElement(Fragment, null, children);
|
||||
|
||||
const DefaultPlayground = ({
|
||||
component,
|
||||
code
|
||||
}) => createElement(Fragment, null, component, code);
|
||||
|
||||
const defaultComponents = {
|
||||
loading: DefaultLoading,
|
||||
playground: DefaultPlayground,
|
||||
notFound: DefaultNotFound,
|
||||
page: DefaultPage
|
||||
};
|
||||
const ctx = createContext({});
|
||||
const ComponentsProvider = ({
|
||||
components: themeComponents = {},
|
||||
children
|
||||
}) => createElement(ctx.Provider, {
|
||||
value: Object.assign({}, defaultComponents, themeComponents)
|
||||
}, children);
|
||||
const useComponents = () => {
|
||||
return useContext(ctx);
|
||||
};
|
||||
|
||||
const isFn = value => typeof value === 'function';
|
||||
function flatArrFromObject(arr, prop) {
|
||||
const reducer = (arr, obj) => {
|
||||
const value = _get(prop)(obj);
|
||||
|
||||
return value ? arr.concat([value]) : arr;
|
||||
};
|
||||
|
||||
return Array.from(new Set(arr.reduce(reducer, [])));
|
||||
}
|
||||
function compare(a, b, reverse) {
|
||||
if (a < b) return reverse ? 1 : -1;
|
||||
if (a > b) return reverse ? -1 : 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
function create(initial) {
|
||||
var _a;
|
||||
|
||||
const ctx = createContext(initial);
|
||||
const listeners = new Set();
|
||||
|
||||
const dispatch = fn => {
|
||||
listeners.forEach(listener => listener(fn));
|
||||
};
|
||||
|
||||
return {
|
||||
context: ctx,
|
||||
set: fn => dispatch(fn),
|
||||
Provider: (_a = class Provider extends Component {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.state = this.props.initial || initial || {};
|
||||
}
|
||||
|
||||
static getDerivedStateFromProps(props, state) {
|
||||
if (!equal(props.initial, state)) return props.initial;
|
||||
return null;
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
listeners.add(fn => this.setState(fn));
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
listeners.clear();
|
||||
}
|
||||
|
||||
render() {
|
||||
return createElement(ctx.Provider, {
|
||||
value: this.state
|
||||
}, this.props.children);
|
||||
}
|
||||
|
||||
}, _a.displayName = 'DoczStateProvider', _a)
|
||||
};
|
||||
}
|
||||
|
||||
const doczState = create({});
|
||||
|
||||
const useConfig = () => {
|
||||
const state = useContext(doczState.context);
|
||||
const {
|
||||
linkComponent,
|
||||
transform,
|
||||
config,
|
||||
themeConfig = {}
|
||||
} = state;
|
||||
|
||||
const newConfig = _merge(themeConfig, config ? config.themeConfig : {});
|
||||
|
||||
const transformed = transform ? transform(newConfig) : newConfig;
|
||||
return Object.assign({}, config, {
|
||||
linkComponent,
|
||||
themeConfig: transformed
|
||||
});
|
||||
};
|
||||
|
||||
const updateState = ev => {
|
||||
const {
|
||||
type,
|
||||
payload
|
||||
} = JSON.parse(ev.data);
|
||||
const prop = type.startsWith('state.') && type.split('.')[1];
|
||||
|
||||
if (prop) {
|
||||
doczState.set(state => Object.assign({}, state, {
|
||||
[prop]: payload
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
const useDataServer = url => {
|
||||
useEffect(() => {
|
||||
if (!url) return;
|
||||
const socket = new WebSocket(url);
|
||||
socket.onmessage = updateState;
|
||||
return () => socket.close();
|
||||
}, []);
|
||||
};
|
||||
|
||||
const useDocs = () => {
|
||||
const {
|
||||
entries = []
|
||||
} = useContext(doczState.context);
|
||||
const arr = entries.map(({
|
||||
value
|
||||
}) => value);
|
||||
return sort(arr, (a, b) => compare(a.name, b.name));
|
||||
};
|
||||
|
||||
const noMenu = entry => !entry.menu;
|
||||
|
||||
const fromMenu = menu => entry => entry.menu === menu;
|
||||
|
||||
const entryAsMenu = entry => ({
|
||||
name: entry.name,
|
||||
route: entry.route,
|
||||
parent: entry.parent
|
||||
});
|
||||
|
||||
const entriesOfMenu = (menu, entries) => entries.filter(fromMenu(menu)).map(entryAsMenu);
|
||||
|
||||
const parseMenu = entries => name => ({
|
||||
name,
|
||||
menu: entriesOfMenu(name, entries)
|
||||
});
|
||||
|
||||
const menusFromEntries = entries => {
|
||||
const entriesWithoutMenu = entries.filter(noMenu).map(entryAsMenu);
|
||||
const menus = flatArrFromObject(entries, 'menu').map(parseMenu(entries));
|
||||
return _unionBy('name', menus, entriesWithoutMenu);
|
||||
};
|
||||
|
||||
const parseItemStr = item => typeof item === 'string' ? {
|
||||
name: item
|
||||
} : item;
|
||||
|
||||
const normalize = item => {
|
||||
const selected = parseItemStr(item);
|
||||
return Object.assign({}, selected, {
|
||||
id: selected.id || ulid(),
|
||||
parent: _get('parent', selected) || _get('parent', item),
|
||||
menu: Array.isArray(selected.menu) ? selected.menu.map(normalize) : selected.menu
|
||||
});
|
||||
};
|
||||
|
||||
const clean = item => item.href || item.route ? _omit('menu', item) : item;
|
||||
|
||||
const normalizeAndClean = _pipe(normalize, clean);
|
||||
|
||||
const mergeMenus = (entriesMenu, configMenu) => {
|
||||
const first = entriesMenu.map(normalizeAndClean);
|
||||
const second = configMenu.map(normalizeAndClean);
|
||||
|
||||
const merged = _unionBy('name', first, second);
|
||||
|
||||
return merged.map(item => {
|
||||
if (!item.menu) return item;
|
||||
const found = second.find(i => i.name === item.name);
|
||||
const foundMenu = found && found.menu;
|
||||
return Object.assign({}, item, {
|
||||
menu: foundMenu ? mergeMenus(item.menu, foundMenu) : item.menu || found.menu
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const UNKNOWN_POS = Infinity;
|
||||
|
||||
const findPos = (item, orderedList = []) => {
|
||||
const name = typeof item !== 'string' ? _get('name', item) : item;
|
||||
const pos = orderedList.findIndex(item => item === name);
|
||||
return pos !== -1 ? pos : UNKNOWN_POS;
|
||||
};
|
||||
|
||||
const compareWithMenu = (to = []) => (a, b) => {
|
||||
const list = to.map(i => i.name || i);
|
||||
return compare(findPos(a, list), findPos(b, list));
|
||||
};
|
||||
|
||||
const sortByName = (a, b) => {
|
||||
return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
|
||||
};
|
||||
|
||||
const sortMenus = (first, second = []) => {
|
||||
const sorted = sort(first, compareWithMenu(second), sortByName);
|
||||
return sorted.map(item => {
|
||||
if (!item.menu) return item;
|
||||
const found = second.find(menu => menu.name === item.name);
|
||||
const foundMenu = found && found.menu;
|
||||
return Object.assign({}, item, {
|
||||
menu: foundMenu ? sortMenus(item.menu, foundMenu) : sort(item.menu, sortByName)
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const search = (val, menu) => {
|
||||
const items = menu.map(item => [item].concat(item.menu || []));
|
||||
|
||||
const flattened = _flattenDepth(2, items);
|
||||
|
||||
const flattenedDeduplicated = [...new Set(flattened)];
|
||||
return match(flattenedDeduplicated, val, {
|
||||
keys: ['name']
|
||||
});
|
||||
};
|
||||
|
||||
const filterMenus = (items, filter) => {
|
||||
if (!filter) return items;
|
||||
return items.filter(filter).map(item => {
|
||||
if (!item.menu) return item;
|
||||
return Object.assign({}, item, {
|
||||
menu: item.menu.filter(filter)
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const useMenus = opts => {
|
||||
const {
|
||||
query = ''
|
||||
} = opts || {};
|
||||
const {
|
||||
entries,
|
||||
config
|
||||
} = useContext(doczState.context);
|
||||
if (!entries) return null;
|
||||
const arr = entries.map(({
|
||||
value
|
||||
}) => value);
|
||||
const entriesMenu = menusFromEntries(arr);
|
||||
const sorted = useMemo(() => {
|
||||
const merged = mergeMenus(entriesMenu, config.menu);
|
||||
const result = sortMenus(merged, config.menu);
|
||||
return filterMenus(result, opts && opts.filter);
|
||||
}, [entries, config]);
|
||||
return query && query.length > 0 ? search(query, sorted) : sorted;
|
||||
};
|
||||
|
||||
const usePrevious = (value, defaultValue) => {
|
||||
const ref = useRef(defaultValue);
|
||||
useEffect(() => {
|
||||
ref.current = value;
|
||||
});
|
||||
return ref.current;
|
||||
};
|
||||
|
||||
const isClient = typeof window === 'object';
|
||||
|
||||
const getSize = (initialWidth, initialHeight) => ({
|
||||
innerHeight: isClient ? window.innerHeight : initialHeight,
|
||||
innerWidth: isClient ? window.innerWidth : initialWidth,
|
||||
outerHeight: isClient ? window.outerHeight : initialHeight,
|
||||
outerWidth: isClient ? window.outerWidth : initialWidth
|
||||
});
|
||||
|
||||
const useWindowSize = (throttleMs = 300, initialWidth = Infinity, initialHeight = Infinity) => {
|
||||
const [windowSize, setWindowSize] = useState(getSize(initialHeight, initialHeight));
|
||||
|
||||
const tSetWindowResize = _throttle(throttleMs, () => setWindowSize(getSize(initialHeight, initialHeight)));
|
||||
|
||||
useEffect(() => {
|
||||
window.addEventListener('resize', tSetWindowResize);
|
||||
return () => void window.removeEventListener('resize', tSetWindowResize);
|
||||
}, []);
|
||||
return windowSize;
|
||||
};
|
||||
|
||||
export { isFn as a, useComponents as b, doczState as c, useConfig as d, useDataServer as e, useDocs as f, useMenus as g, usePrevious as h, useWindowSize as i, ComponentsProvider as j };
|
||||
|
|
@ -0,0 +1,324 @@
|
|||
'use strict';
|
||||
|
||||
function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }
|
||||
|
||||
var React = require('react');
|
||||
var _get = _interopDefault(require('lodash/fp/get'));
|
||||
var _omit = _interopDefault(require('lodash/fp/omit'));
|
||||
var equal = _interopDefault(require('fast-deep-equal'));
|
||||
var _merge = _interopDefault(require('lodash/fp/merge'));
|
||||
var sort = _interopDefault(require('array-sort'));
|
||||
var _unionBy = _interopDefault(require('lodash/fp/unionBy'));
|
||||
var _flattenDepth = _interopDefault(require('lodash/fp/flattenDepth'));
|
||||
var _pipe = _interopDefault(require('lodash/fp/pipe'));
|
||||
var ulid = require('ulid');
|
||||
var match = _interopDefault(require('match-sorter'));
|
||||
var _throttle = _interopDefault(require('lodash/fp/throttle'));
|
||||
|
||||
const DefaultNotFound = () => React.createElement(React.Fragment, null, "Not found");
|
||||
|
||||
const DefaultLoading = () => React.createElement(React.Fragment, null, "Loading");
|
||||
|
||||
const DefaultPage = ({
|
||||
children
|
||||
}) => React.createElement(React.Fragment, null, children);
|
||||
|
||||
const DefaultPlayground = ({
|
||||
component,
|
||||
code
|
||||
}) => React.createElement(React.Fragment, null, component, code);
|
||||
|
||||
const defaultComponents = {
|
||||
loading: DefaultLoading,
|
||||
playground: DefaultPlayground,
|
||||
notFound: DefaultNotFound,
|
||||
page: DefaultPage
|
||||
};
|
||||
const ctx = React.createContext({});
|
||||
const ComponentsProvider = ({
|
||||
components: themeComponents = {},
|
||||
children
|
||||
}) => React.createElement(ctx.Provider, {
|
||||
value: Object.assign({}, defaultComponents, themeComponents)
|
||||
}, children);
|
||||
const useComponents = () => {
|
||||
return React.useContext(ctx);
|
||||
};
|
||||
|
||||
const isFn = value => typeof value === 'function';
|
||||
function flatArrFromObject(arr, prop) {
|
||||
const reducer = (arr, obj) => {
|
||||
const value = _get(prop)(obj);
|
||||
|
||||
return value ? arr.concat([value]) : arr;
|
||||
};
|
||||
|
||||
return Array.from(new Set(arr.reduce(reducer, [])));
|
||||
}
|
||||
function compare(a, b, reverse) {
|
||||
if (a < b) return reverse ? 1 : -1;
|
||||
if (a > b) return reverse ? -1 : 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
function create(initial) {
|
||||
var _a;
|
||||
|
||||
const ctx = React.createContext(initial);
|
||||
const listeners = new Set();
|
||||
|
||||
const dispatch = fn => {
|
||||
listeners.forEach(listener => listener(fn));
|
||||
};
|
||||
|
||||
return {
|
||||
context: ctx,
|
||||
set: fn => dispatch(fn),
|
||||
Provider: (_a = class Provider extends React.Component {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.state = this.props.initial || initial || {};
|
||||
}
|
||||
|
||||
static getDerivedStateFromProps(props, state) {
|
||||
if (!equal(props.initial, state)) return props.initial;
|
||||
return null;
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
listeners.add(fn => this.setState(fn));
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
listeners.clear();
|
||||
}
|
||||
|
||||
render() {
|
||||
return React.createElement(ctx.Provider, {
|
||||
value: this.state
|
||||
}, this.props.children);
|
||||
}
|
||||
|
||||
}, _a.displayName = 'DoczStateProvider', _a)
|
||||
};
|
||||
}
|
||||
|
||||
const doczState = create({});
|
||||
|
||||
const useConfig = () => {
|
||||
const state = React.useContext(doczState.context);
|
||||
const {
|
||||
linkComponent,
|
||||
transform,
|
||||
config,
|
||||
themeConfig = {}
|
||||
} = state;
|
||||
|
||||
const newConfig = _merge(themeConfig, config ? config.themeConfig : {});
|
||||
|
||||
const transformed = transform ? transform(newConfig) : newConfig;
|
||||
return Object.assign({}, config, {
|
||||
linkComponent,
|
||||
themeConfig: transformed
|
||||
});
|
||||
};
|
||||
|
||||
const updateState = ev => {
|
||||
const {
|
||||
type,
|
||||
payload
|
||||
} = JSON.parse(ev.data);
|
||||
const prop = type.startsWith('state.') && type.split('.')[1];
|
||||
|
||||
if (prop) {
|
||||
doczState.set(state => Object.assign({}, state, {
|
||||
[prop]: payload
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
const useDataServer = url => {
|
||||
React.useEffect(() => {
|
||||
if (!url) return;
|
||||
const socket = new WebSocket(url);
|
||||
socket.onmessage = updateState;
|
||||
return () => socket.close();
|
||||
}, []);
|
||||
};
|
||||
|
||||
const useDocs = () => {
|
||||
const {
|
||||
entries = []
|
||||
} = React.useContext(doczState.context);
|
||||
const arr = entries.map(({
|
||||
value
|
||||
}) => value);
|
||||
return sort(arr, (a, b) => compare(a.name, b.name));
|
||||
};
|
||||
|
||||
const noMenu = entry => !entry.menu;
|
||||
|
||||
const fromMenu = menu => entry => entry.menu === menu;
|
||||
|
||||
const entryAsMenu = entry => ({
|
||||
name: entry.name,
|
||||
route: entry.route,
|
||||
parent: entry.parent
|
||||
});
|
||||
|
||||
const entriesOfMenu = (menu, entries) => entries.filter(fromMenu(menu)).map(entryAsMenu);
|
||||
|
||||
const parseMenu = entries => name => ({
|
||||
name,
|
||||
menu: entriesOfMenu(name, entries)
|
||||
});
|
||||
|
||||
const menusFromEntries = entries => {
|
||||
const entriesWithoutMenu = entries.filter(noMenu).map(entryAsMenu);
|
||||
const menus = flatArrFromObject(entries, 'menu').map(parseMenu(entries));
|
||||
return _unionBy('name', menus, entriesWithoutMenu);
|
||||
};
|
||||
|
||||
const parseItemStr = item => typeof item === 'string' ? {
|
||||
name: item
|
||||
} : item;
|
||||
|
||||
const normalize = item => {
|
||||
const selected = parseItemStr(item);
|
||||
return Object.assign({}, selected, {
|
||||
id: selected.id || ulid.ulid(),
|
||||
parent: _get('parent', selected) || _get('parent', item),
|
||||
menu: Array.isArray(selected.menu) ? selected.menu.map(normalize) : selected.menu
|
||||
});
|
||||
};
|
||||
|
||||
const clean = item => item.href || item.route ? _omit('menu', item) : item;
|
||||
|
||||
const normalizeAndClean = _pipe(normalize, clean);
|
||||
|
||||
const mergeMenus = (entriesMenu, configMenu) => {
|
||||
const first = entriesMenu.map(normalizeAndClean);
|
||||
const second = configMenu.map(normalizeAndClean);
|
||||
|
||||
const merged = _unionBy('name', first, second);
|
||||
|
||||
return merged.map(item => {
|
||||
if (!item.menu) return item;
|
||||
const found = second.find(i => i.name === item.name);
|
||||
const foundMenu = found && found.menu;
|
||||
return Object.assign({}, item, {
|
||||
menu: foundMenu ? mergeMenus(item.menu, foundMenu) : item.menu || found.menu
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const UNKNOWN_POS = Infinity;
|
||||
|
||||
const findPos = (item, orderedList = []) => {
|
||||
const name = typeof item !== 'string' ? _get('name', item) : item;
|
||||
const pos = orderedList.findIndex(item => item === name);
|
||||
return pos !== -1 ? pos : UNKNOWN_POS;
|
||||
};
|
||||
|
||||
const compareWithMenu = (to = []) => (a, b) => {
|
||||
const list = to.map(i => i.name || i);
|
||||
return compare(findPos(a, list), findPos(b, list));
|
||||
};
|
||||
|
||||
const sortByName = (a, b) => {
|
||||
return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
|
||||
};
|
||||
|
||||
const sortMenus = (first, second = []) => {
|
||||
const sorted = sort(first, compareWithMenu(second), sortByName);
|
||||
return sorted.map(item => {
|
||||
if (!item.menu) return item;
|
||||
const found = second.find(menu => menu.name === item.name);
|
||||
const foundMenu = found && found.menu;
|
||||
return Object.assign({}, item, {
|
||||
menu: foundMenu ? sortMenus(item.menu, foundMenu) : sort(item.menu, sortByName)
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const search = (val, menu) => {
|
||||
const items = menu.map(item => [item].concat(item.menu || []));
|
||||
|
||||
const flattened = _flattenDepth(2, items);
|
||||
|
||||
const flattenedDeduplicated = [...new Set(flattened)];
|
||||
return match(flattenedDeduplicated, val, {
|
||||
keys: ['name']
|
||||
});
|
||||
};
|
||||
|
||||
const filterMenus = (items, filter) => {
|
||||
if (!filter) return items;
|
||||
return items.filter(filter).map(item => {
|
||||
if (!item.menu) return item;
|
||||
return Object.assign({}, item, {
|
||||
menu: item.menu.filter(filter)
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const useMenus = opts => {
|
||||
const {
|
||||
query = ''
|
||||
} = opts || {};
|
||||
const {
|
||||
entries,
|
||||
config
|
||||
} = React.useContext(doczState.context);
|
||||
if (!entries) return null;
|
||||
const arr = entries.map(({
|
||||
value
|
||||
}) => value);
|
||||
const entriesMenu = menusFromEntries(arr);
|
||||
const sorted = React.useMemo(() => {
|
||||
const merged = mergeMenus(entriesMenu, config.menu);
|
||||
const result = sortMenus(merged, config.menu);
|
||||
return filterMenus(result, opts && opts.filter);
|
||||
}, [entries, config]);
|
||||
return query && query.length > 0 ? search(query, sorted) : sorted;
|
||||
};
|
||||
|
||||
const usePrevious = (value, defaultValue) => {
|
||||
const ref = React.useRef(defaultValue);
|
||||
React.useEffect(() => {
|
||||
ref.current = value;
|
||||
});
|
||||
return ref.current;
|
||||
};
|
||||
|
||||
const isClient = typeof window === 'object';
|
||||
|
||||
const getSize = (initialWidth, initialHeight) => ({
|
||||
innerHeight: isClient ? window.innerHeight : initialHeight,
|
||||
innerWidth: isClient ? window.innerWidth : initialWidth,
|
||||
outerHeight: isClient ? window.outerHeight : initialHeight,
|
||||
outerWidth: isClient ? window.outerWidth : initialWidth
|
||||
});
|
||||
|
||||
const useWindowSize = (throttleMs = 300, initialWidth = Infinity, initialHeight = Infinity) => {
|
||||
const [windowSize, setWindowSize] = React.useState(getSize(initialHeight, initialHeight));
|
||||
|
||||
const tSetWindowResize = _throttle(throttleMs, () => setWindowSize(getSize(initialHeight, initialHeight)));
|
||||
|
||||
React.useEffect(() => {
|
||||
window.addEventListener('resize', tSetWindowResize);
|
||||
return () => void window.removeEventListener('resize', tSetWindowResize);
|
||||
}, []);
|
||||
return windowSize;
|
||||
};
|
||||
|
||||
exports.ComponentsProvider = ComponentsProvider;
|
||||
exports.doczState = doczState;
|
||||
exports.isFn = isFn;
|
||||
exports.useComponents = useComponents;
|
||||
exports.useConfig = useConfig;
|
||||
exports.useDataServer = useDataServer;
|
||||
exports.useDocs = useDocs;
|
||||
exports.useMenus = useMenus;
|
||||
exports.usePrevious = usePrevious;
|
||||
exports.useWindowSize = useWindowSize;
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
import { SFC, ComponentType } from 'react';
|
||||
interface Props {
|
||||
as: ComponentType<any>;
|
||||
getInitialProps?: (props: any) => any;
|
||||
}
|
||||
export declare const AsyncComponent: SFC<Props>;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
import * as React from 'react';
|
||||
export declare const Playground: React.SFC<any>;
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import { SFC } from 'react';
|
||||
import { Entry } from '../state';
|
||||
import { ComponentsMap } from '../hooks/useComponents';
|
||||
export declare type Imports = Record<string, () => Promise<any>>;
|
||||
export declare const loadRoute: (path: string, imports: Record<string, () => Promise<any>>, components: ComponentsMap) => import("@loadable/component").LoadableComponent<any>;
|
||||
interface AsyncRouteProps {
|
||||
asyncComponent: any;
|
||||
components: ComponentsMap;
|
||||
path: string;
|
||||
entry: Entry;
|
||||
}
|
||||
export declare const AsyncRoute: SFC<AsyncRouteProps>;
|
||||
export {};
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
import * as React from 'react';
|
||||
import { LinkProps } from '@reach/router';
|
||||
export { LinkProps };
|
||||
export declare const Link: React.ForwardRefExoticComponent<Pick<LinkProps<any>, "key" | "download" | "hrefLang" | "media" | "rel" | "target" | "type" | "referrerPolicy" | "defaultChecked" | "defaultValue" | "suppressContentEditableWarning" | "suppressHydrationWarning" | "accessKey" | "className" | "contentEditable" | "contextMenu" | "dir" | "draggable" | "hidden" | "id" | "lang" | "placeholder" | "slot" | "spellCheck" | "style" | "tabIndex" | "title" | "inputMode" | "is" | "radioGroup" | "role" | "about" | "datatype" | "inlist" | "prefix" | "property" | "resource" | "typeof" | "vocab" | "autoCapitalize" | "autoCorrect" | "autoSave" | "color" | "itemProp" | "itemScope" | "itemType" | "itemID" | "itemRef" | "results" | "security" | "unselectable" | "aria-activedescendant" | "aria-atomic" | "aria-autocomplete" | "aria-busy" | "aria-checked" | "aria-colcount" | "aria-colindex" | "aria-colspan" | "aria-controls" | "aria-current" | "aria-describedby" | "aria-details" | "aria-disabled" | "aria-dropeffect" | "aria-errormessage" | "aria-expanded" | "aria-flowto" | "aria-grabbed" | "aria-haspopup" | "aria-hidden" | "aria-invalid" | "aria-keyshortcuts" | "aria-label" | "aria-labelledby" | "aria-level" | "aria-live" | "aria-modal" | "aria-multiline" | "aria-multiselectable" | "aria-orientation" | "aria-owns" | "aria-placeholder" | "aria-posinset" | "aria-pressed" | "aria-readonly" | "aria-relevant" | "aria-required" | "aria-roledescription" | "aria-rowcount" | "aria-rowindex" | "aria-rowspan" | "aria-selected" | "aria-setsize" | "aria-sort" | "aria-valuemax" | "aria-valuemin" | "aria-valuenow" | "aria-valuetext" | "children" | "dangerouslySetInnerHTML" | "onCopy" | "onCopyCapture" | "onCut" | "onCutCapture" | "onPaste" | "onPasteCapture" | "onCompositionEnd" | "onCompositionEndCapture" | "onCompositionStart" | "onCompositionStartCapture" | "onCompositionUpdate" | "onCompositionUpdateCapture" | "onFocus" | "onFocusCapture" | "onBlur" | "onBlurCapture" | "onChange" | "onChangeCapture" | "onBeforeInput" | "onBeforeInputCapture" | "onInput" | "onInputCapture" | "onReset" | "onResetCapture" | "onSubmit" | "onSubmitCapture" | "onInvalid" | "onInvalidCapture" | "onLoad" | "onLoadCapture" | "onError" | "onErrorCapture" | "onKeyDown" | "onKeyDownCapture" | "onKeyPress" | "onKeyPressCapture" | "onKeyUp" | "onKeyUpCapture" | "onAbort" | "onAbortCapture" | "onCanPlay" | "onCanPlayCapture" | "onCanPlayThrough" | "onCanPlayThroughCapture" | "onDurationChange" | "onDurationChangeCapture" | "onEmptied" | "onEmptiedCapture" | "onEncrypted" | "onEncryptedCapture" | "onEnded" | "onEndedCapture" | "onLoadedData" | "onLoadedDataCapture" | "onLoadedMetadata" | "onLoadedMetadataCapture" | "onLoadStart" | "onLoadStartCapture" | "onPause" | "onPauseCapture" | "onPlay" | "onPlayCapture" | "onPlaying" | "onPlayingCapture" | "onProgress" | "onProgressCapture" | "onRateChange" | "onRateChangeCapture" | "onSeeked" | "onSeekedCapture" | "onSeeking" | "onSeekingCapture" | "onStalled" | "onStalledCapture" | "onSuspend" | "onSuspendCapture" | "onTimeUpdate" | "onTimeUpdateCapture" | "onVolumeChange" | "onVolumeChangeCapture" | "onWaiting" | "onWaitingCapture" | "onAuxClick" | "onAuxClickCapture" | "onClick" | "onClickCapture" | "onContextMenu" | "onContextMenuCapture" | "onDoubleClick" | "onDoubleClickCapture" | "onDrag" | "onDragCapture" | "onDragEnd" | "onDragEndCapture" | "onDragEnter" | "onDragEnterCapture" | "onDragExit" | "onDragExitCapture" | "onDragLeave" | "onDragLeaveCapture" | "onDragOver" | "onDragOverCapture" | "onDragStart" | "onDragStartCapture" | "onDrop" | "onDropCapture" | "onMouseDown" | "onMouseDownCapture" | "onMouseEnter" | "onMouseLeave" | "onMouseMove" | "onMouseMoveCapture" | "onMouseOut" | "onMouseOutCapture" | "onMouseOver" | "onMouseOverCapture" | "onMouseUp" | "onMouseUpCapture" | "onSelect" | "onSelectCapture" | "onTouchCancel" | "onTouchCancelCapture" | "onTouchEnd" | "onTouchEndCapture" | "onTouchMove" | "onTouchMoveCapture" | "onTouchStart" | "onTouchStartCapture" | "onPointerDown" | "onPointerDownCapture" | "onPointerMove" | "onPointerMoveCapture" | "onPointerUp" | "onPointerUpCapture" | "onPointerCancel" | "onPointerCancelCapture" | "onPointerEnter" | "onPointerEnterCapture" | "onPointerLeave" | "onPointerLeaveCapture" | "onPointerOver" | "onPointerOverCapture" | "onPointerOut" | "onPointerOutCapture" | "onGotPointerCapture" | "onGotPointerCaptureCapture" | "onLostPointerCapture" | "onLostPointerCaptureCapture" | "onScroll" | "onScrollCapture" | "onWheel" | "onWheelCapture" | "onAnimationStart" | "onAnimationStartCapture" | "onAnimationEnd" | "onAnimationEndCapture" | "onAnimationIteration" | "onAnimationIterationCapture" | "onTransitionEnd" | "onTransitionEndCapture" | "to" | "replace" | "getProps" | "state"> & React.RefAttributes<any>>;
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import { ComponentType, SFC } from 'react';
|
||||
export interface PlaygroundProps {
|
||||
className?: string;
|
||||
style?: any;
|
||||
wrapper?: ComponentType<any>;
|
||||
children: any;
|
||||
__scope: Record<string, any>;
|
||||
__position: number;
|
||||
__code: string;
|
||||
__codesandbox: string;
|
||||
}
|
||||
declare const Playground: SFC<PlaygroundProps>;
|
||||
export default Playground;
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
import { SFC, ComponentType } from 'react';
|
||||
export interface EnumValue {
|
||||
value: string;
|
||||
computed: boolean;
|
||||
}
|
||||
export interface FlowTypeElement {
|
||||
name: string;
|
||||
value: string;
|
||||
}
|
||||
export interface FlowTypeArgs {
|
||||
name: string;
|
||||
type: {
|
||||
name: string;
|
||||
};
|
||||
}
|
||||
export interface PropType {
|
||||
name: string;
|
||||
value?: any;
|
||||
raw?: any;
|
||||
computed?: boolean;
|
||||
}
|
||||
export interface FlowType extends PropType {
|
||||
elements: FlowTypeElement[];
|
||||
name: string;
|
||||
raw: string;
|
||||
type?: string;
|
||||
computed?: boolean;
|
||||
signature?: {
|
||||
arguments: FlowTypeArgs[];
|
||||
return: {
|
||||
name: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
export interface Prop {
|
||||
required: boolean;
|
||||
description?: string;
|
||||
type: PropType;
|
||||
defaultValue?: {
|
||||
value: string;
|
||||
computed: boolean;
|
||||
};
|
||||
flowType?: FlowType;
|
||||
}
|
||||
export declare type ComponentWithDocGenInfo = ComponentType & {
|
||||
__docgenInfo: {
|
||||
description?: string;
|
||||
props?: Record<string, Prop>;
|
||||
};
|
||||
};
|
||||
export interface PropsProps {
|
||||
title?: Node;
|
||||
isRaw?: boolean;
|
||||
isToggle?: boolean;
|
||||
of: ComponentWithDocGenInfo;
|
||||
}
|
||||
export declare const getPropType: (prop: Prop) => any;
|
||||
export interface PropsComponentProps {
|
||||
title?: Node;
|
||||
isRaw?: boolean;
|
||||
isToggle?: boolean;
|
||||
props: Record<string, Prop>;
|
||||
getPropType(prop: Prop): string;
|
||||
}
|
||||
export declare const Props: SFC<PropsProps>;
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { SFC } from 'react';
|
||||
import { ComponentsMap } from '../hooks/useComponents';
|
||||
import { Imports } from './AsyncRoute';
|
||||
export interface RoutesProps {
|
||||
components: ComponentsMap;
|
||||
imports: Imports;
|
||||
}
|
||||
export declare const Routes: SFC<RoutesProps>;
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
export * from './useComponents';
|
||||
export { useConfig, UseConfigObj } from './useConfig';
|
||||
export { useDataServer } from './useDataServer';
|
||||
export { useDocs } from './useDocs';
|
||||
export { useMenus } from './useMenus';
|
||||
export { usePrevious } from './usePrevious';
|
||||
export { useWindowSize } from './useWindowSize';
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import { SFC, ComponentType as CT } from 'react';
|
||||
import { RouteComponentProps } from '@reach/router';
|
||||
import { Entry } from '../state';
|
||||
export declare type PageProps = RouteComponentProps<any> & {
|
||||
doc: Entry;
|
||||
};
|
||||
export interface PlaygroundProps {
|
||||
className?: string;
|
||||
style?: any;
|
||||
wrapper?: CT<any>;
|
||||
components: ComponentsMap;
|
||||
component: JSX.Element;
|
||||
position: number;
|
||||
code: string;
|
||||
codesandbox: string;
|
||||
scope: Record<string, any>;
|
||||
}
|
||||
export declare type PlaygroundComponent = CT<PlaygroundProps>;
|
||||
export interface ComponentsMap {
|
||||
loading?: CT;
|
||||
page?: CT<PageProps>;
|
||||
notFound?: CT<RouteComponentProps<any>>;
|
||||
playground?: PlaygroundComponent;
|
||||
h1?: CT<any> | string;
|
||||
h2?: CT<any> | string;
|
||||
h3?: CT<any> | string;
|
||||
h4?: CT<any> | string;
|
||||
h5?: CT<any> | string;
|
||||
h6?: CT<any> | string;
|
||||
span?: CT<any> | string;
|
||||
a?: CT<any> | string;
|
||||
ul?: CT<any> | string;
|
||||
table?: CT<any> | string;
|
||||
pre?: CT<any> | string;
|
||||
code?: CT<any> | string;
|
||||
inlineCode?: CT<any> | string;
|
||||
[key: string]: any;
|
||||
}
|
||||
export declare type NotFoundComponent = CT<RouteComponentProps<any>>;
|
||||
export interface ComponentsProviderProps {
|
||||
components: ComponentsMap;
|
||||
}
|
||||
export declare const ComponentsProvider: SFC<ComponentsProviderProps>;
|
||||
export declare const useComponents: () => ComponentsMap;
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
/// <reference types="react" />
|
||||
import { ThemeConfig, Config } from '../state';
|
||||
export interface UseConfigObj extends Config {
|
||||
themeConfig: ThemeConfig;
|
||||
linkComponent?: React.ComponentType<any>;
|
||||
}
|
||||
export declare const useConfig: () => UseConfigObj;
|
||||
|
|
@ -0,0 +1 @@
|
|||
export declare const useDataServer: (url: string | undefined) => void;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
import { Entry } from '../state';
|
||||
export declare const useDocs: () => Entry[] | null;
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { MenuItem } from '../state';
|
||||
declare type FilterFn = (item: MenuItem) => boolean;
|
||||
export interface UseMenusParams {
|
||||
query?: string;
|
||||
filter?: FilterFn;
|
||||
}
|
||||
export declare const useMenus: (opts?: UseMenusParams | undefined) => MenuItem[] | null;
|
||||
export {};
|
||||
|
|
@ -0,0 +1 @@
|
|||
export declare const usePrevious: (value: any, defaultValue?: any) => any;
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
export declare const useWindowSize: (throttleMs?: number, initialWidth?: number, initialHeight?: number) => {
|
||||
innerHeight: number;
|
||||
innerWidth: number;
|
||||
outerHeight: number;
|
||||
outerWidth: number;
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
export { Playground } from './components/AsyncPlayground';
|
||||
export { AsyncRoute, loadRoute } from './components/AsyncRoute';
|
||||
export { Link, LinkProps } from './components/Link';
|
||||
export { Props, PropsComponentProps } from './components/Props';
|
||||
export { Routes } from './components/Routes';
|
||||
export * from './hooks';
|
||||
export { theme } from './theme';
|
||||
export { doczState, Entry, MenuItem, ThemeConfig } from './state';
|
||||
|
|
@ -0,0 +1,302 @@
|
|||
import * as React from 'react';
|
||||
import { createElement, Suspense, useState, useEffect, forwardRef, useCallback, useContext, useMemo, memo, Fragment } from 'react';
|
||||
import loadable from '@loadable/component';
|
||||
import _get from 'lodash/fp/get';
|
||||
import { __rest } from 'tslib';
|
||||
import { a as isFn, b as useComponents, c as doczState } from './chunk.esm.js';
|
||||
export { j as ComponentsProvider, c as doczState, b as useComponents, d as useConfig, e as useDataServer, f as useDocs, g as useMenus, h as usePrevious, i as useWindowSize } from './chunk.esm.js';
|
||||
import _omit from 'lodash/fp/omit';
|
||||
import { Link as Link$1, createHistory, LocationProvider, Router } from '@reach/router';
|
||||
import _first from 'lodash/fp/first';
|
||||
import capitalize from 'capitalize';
|
||||
import 'fast-deep-equal';
|
||||
import 'lodash/fp/merge';
|
||||
import 'array-sort';
|
||||
import 'lodash/fp/unionBy';
|
||||
import 'lodash/fp/flattenDepth';
|
||||
import 'lodash/fp/pipe';
|
||||
import 'ulid';
|
||||
import 'match-sorter';
|
||||
import 'lodash/fp/throttle';
|
||||
import { MDXProvider } from '@mdx-js/react';
|
||||
import createHashSource from 'hash-source';
|
||||
let source = createHashSource();
|
||||
|
||||
const BasePlayground = loadable(() => import('./Playground.esm.js'));
|
||||
const Playground = props => typeof window !== 'undefined' ? createElement(Suspense, {
|
||||
fallback: null
|
||||
}, createElement(BasePlayground, Object.assign({}, props))) : null;
|
||||
|
||||
const AsyncComponent = defaultProps => {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState(null);
|
||||
const [data, setData] = useState({});
|
||||
useEffect(() => {
|
||||
const {
|
||||
getInitialProps
|
||||
} = defaultProps;
|
||||
|
||||
if (getInitialProps && isFn(getInitialProps)) {
|
||||
setLoading(true);
|
||||
getInitialProps(defaultProps).then(data => {
|
||||
setLoading(false);
|
||||
setError(null);
|
||||
setData(data);
|
||||
}).catch(err => {
|
||||
setLoading(false);
|
||||
setError(err);
|
||||
setData({});
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
const {
|
||||
as: Comp,
|
||||
getInitialProps
|
||||
} = defaultProps,
|
||||
props = __rest(defaultProps, ["as", "getInitialProps"]);
|
||||
|
||||
return createElement(Comp, Object.assign({}, props, {
|
||||
data: Object.assign({}, data, {
|
||||
loading,
|
||||
error
|
||||
})
|
||||
}));
|
||||
};
|
||||
|
||||
const loadRoute = (path, imports, components) => {
|
||||
const Loading = components.loading;
|
||||
|
||||
const fn = async () => {
|
||||
const importFn = _get(path, imports);
|
||||
|
||||
const {
|
||||
default: Component,
|
||||
getInitialProps
|
||||
} = await importFn();
|
||||
|
||||
const ExportedComponent = props => createElement(AsyncComponent, Object.assign({}, props, {
|
||||
as: Component || 'div',
|
||||
getInitialProps: getInitialProps
|
||||
}));
|
||||
|
||||
return ExportedComponent;
|
||||
};
|
||||
|
||||
return loadable(fn, {
|
||||
fallback: createElement(Loading, null)
|
||||
});
|
||||
};
|
||||
const AsyncRoute = defaultProps => {
|
||||
const {
|
||||
asyncComponent,
|
||||
path,
|
||||
entry
|
||||
} = defaultProps,
|
||||
routeProps = __rest(defaultProps, ["asyncComponent", "path", "entry"]);
|
||||
|
||||
const components = useComponents();
|
||||
const Page = components.page;
|
||||
const Component = asyncComponent;
|
||||
const props = Object.assign({}, routeProps, {
|
||||
doc: entry
|
||||
});
|
||||
return Page ? createElement(Page, Object.assign({}, props), createElement(Component, Object.assign({}, props))) : createElement(Component, Object.assign({}, props));
|
||||
};
|
||||
|
||||
const Link = forwardRef((defaultProps, ref) => {
|
||||
const props = _omit(['activeClassName', 'partiallyActive'], defaultProps);
|
||||
|
||||
const isActive = useCallback(({
|
||||
isCurrent
|
||||
}) => {
|
||||
return isCurrent ? {
|
||||
className: `${props.className} active`
|
||||
} : {};
|
||||
}, [props.className]);
|
||||
return createElement(Link$1, Object.assign({}, props, {
|
||||
getProps: isActive,
|
||||
ref: ref
|
||||
}));
|
||||
});
|
||||
Link.displayName = 'Link';
|
||||
|
||||
const RE_OBJECTOF = /(?:React\.)?(?:PropTypes\.)?objectOf\((?:React\.)?(?:PropTypes\.)?(\w+)\)/;
|
||||
|
||||
const getTypeStr = type => {
|
||||
switch (type.name.toLowerCase()) {
|
||||
case 'instanceof':
|
||||
return `Class(${type.value})`;
|
||||
|
||||
case 'enum':
|
||||
if (type.computed) return type.value;
|
||||
return type.value ? type.value.map(v => `${v.value}`).join(' │ ') : type.raw;
|
||||
|
||||
case 'union':
|
||||
return type.value ? type.value.map(t => `${getTypeStr(t)}`).join(' │ ') : type.raw;
|
||||
|
||||
case 'array':
|
||||
return type.raw;
|
||||
|
||||
case 'arrayof':
|
||||
return `Array<${getTypeStr(type.value)}>`;
|
||||
|
||||
case 'custom':
|
||||
if (type.raw.indexOf('function') !== -1 || type.raw.indexOf('=>') !== -1) return 'Custom(Function)';else if (type.raw.toLowerCase().indexOf('objectof') !== -1) {
|
||||
const m = type.raw.match(RE_OBJECTOF);
|
||||
if (m && m[1]) return `ObjectOf(${capitalize(m[1])})`;
|
||||
return 'ObjectOf';
|
||||
}
|
||||
return 'Custom';
|
||||
|
||||
case 'bool':
|
||||
return 'Boolean';
|
||||
|
||||
case 'func':
|
||||
return 'Function';
|
||||
|
||||
case 'shape':
|
||||
const shape = type.value;
|
||||
const rst = {};
|
||||
Object.keys(shape).forEach(key => {
|
||||
rst[key] = getTypeStr(shape[key]);
|
||||
});
|
||||
return JSON.stringify(rst, null, 2);
|
||||
|
||||
default:
|
||||
return capitalize(type.name);
|
||||
}
|
||||
};
|
||||
|
||||
const humanize = type => getTypeStr(type);
|
||||
|
||||
const getPropType = prop => {
|
||||
const propName = _get('name', prop.flowType || prop.type);
|
||||
|
||||
if (!propName) return null;
|
||||
const isEnum = propName.startsWith('"') || propName === 'enum';
|
||||
const name = capitalize(isEnum ? 'enum' : propName);
|
||||
|
||||
const value = _get('type.value', prop);
|
||||
|
||||
if (!name) return null;
|
||||
|
||||
if (isEnum && typeof value === 'string' || !prop.flowType && !isEnum && !value || prop.flowType && !prop.flowType.elements) {
|
||||
return name;
|
||||
}
|
||||
|
||||
return prop.flowType ? humanize(prop.flowType) : humanize(prop.type);
|
||||
};
|
||||
const Props = ({
|
||||
title,
|
||||
isToggle,
|
||||
isRaw,
|
||||
of: component
|
||||
}) => {
|
||||
const components = useComponents();
|
||||
const {
|
||||
props: stateProps
|
||||
} = useContext(doczState.context);
|
||||
const PropsComponent = components.props;
|
||||
|
||||
const filename = _get('__filemeta.filename', component);
|
||||
|
||||
const filemetaName = _get('__filemeta.name', component);
|
||||
|
||||
const componentName = filemetaName || component.displayName || component.name;
|
||||
const found = stateProps && stateProps.length > 0 && stateProps.find(item => filename ? item.key === filename : item.key.includes(`${componentName}.`));
|
||||
const value = _get('value', found) || [];
|
||||
|
||||
const firstDefinition = _first(value);
|
||||
|
||||
const definition = value.find(i => i.displayName === componentName);
|
||||
|
||||
const props = _get('props', definition || firstDefinition);
|
||||
|
||||
if (!props) return null;
|
||||
if (!PropsComponent) return null;
|
||||
return createElement(PropsComponent, {
|
||||
title: title,
|
||||
isRaw: isRaw,
|
||||
isToggle: isToggle,
|
||||
props: props,
|
||||
getPropType: getPropType
|
||||
});
|
||||
};
|
||||
|
||||
const goToHash = ({
|
||||
location
|
||||
}) => {
|
||||
setTimeout(() => {
|
||||
if (location && location.hash) {
|
||||
const decodedHash = decodeURI(location.hash);
|
||||
const id = decodedHash.substring(1);
|
||||
const el = document.getElementById(id);
|
||||
if (el) el.scrollIntoView();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const Routes = ({
|
||||
imports
|
||||
}) => {
|
||||
const components = useComponents();
|
||||
const {
|
||||
entries
|
||||
} = useContext(doczState.context);
|
||||
const NotFound = components.notFound;
|
||||
const history = useMemo(() => createHistory(source), []);
|
||||
useEffect(() => {
|
||||
history.listen(goToHash);
|
||||
}, []);
|
||||
return createElement(MDXProvider, {
|
||||
components: components
|
||||
}, createElement(LocationProvider, {
|
||||
history: history
|
||||
}, createElement(Router, null, createElement(NotFound, {
|
||||
default: true
|
||||
}), entries && entries.map(({
|
||||
key: path,
|
||||
value: entry
|
||||
}) => {
|
||||
const props = {
|
||||
path,
|
||||
entries,
|
||||
components
|
||||
};
|
||||
const component = loadRoute(path, imports, components);
|
||||
return createElement(AsyncRoute, Object.assign({}, props, {
|
||||
entry: entry,
|
||||
key: entry.id,
|
||||
path: entry.route,
|
||||
asyncComponent: component
|
||||
}));
|
||||
}))));
|
||||
};
|
||||
|
||||
function theme(themeConfig, transform = c => c) {
|
||||
return WrappedComponent => {
|
||||
const Theme = memo(props => {
|
||||
const {
|
||||
linkComponent
|
||||
} = props;
|
||||
const {
|
||||
db,
|
||||
children,
|
||||
wrapper: Wrapper = Fragment
|
||||
} = props;
|
||||
const initial = Object.assign({}, db, {
|
||||
themeConfig,
|
||||
transform,
|
||||
linkComponent
|
||||
});
|
||||
return createElement(doczState.Provider, {
|
||||
initial: initial
|
||||
}, createElement(Wrapper, null, createElement(WrappedComponent, null, children)));
|
||||
});
|
||||
Theme.displayName = WrappedComponent.displayName || 'DoczTheme';
|
||||
return Theme;
|
||||
};
|
||||
}
|
||||
|
||||
export { AsyncRoute, Link, Playground, Props, Routes, loadRoute, theme };
|
||||
|
|
@ -0,0 +1,321 @@
|
|||
'use strict';
|
||||
|
||||
Object.defineProperty(exports, '__esModule', { value: true });
|
||||
|
||||
function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }
|
||||
|
||||
var React = require('react');
|
||||
var loadable = _interopDefault(require('@loadable/component'));
|
||||
var _get = _interopDefault(require('lodash/fp/get'));
|
||||
var tslib_1 = require('tslib');
|
||||
var __chunk_1 = require('./chunk.js');
|
||||
var _omit = _interopDefault(require('lodash/fp/omit'));
|
||||
var router = require('@reach/router');
|
||||
var _first = _interopDefault(require('lodash/fp/first'));
|
||||
var capitalize = _interopDefault(require('capitalize'));
|
||||
require('fast-deep-equal');
|
||||
require('lodash/fp/merge');
|
||||
require('array-sort');
|
||||
require('lodash/fp/unionBy');
|
||||
require('lodash/fp/flattenDepth');
|
||||
require('lodash/fp/pipe');
|
||||
require('ulid');
|
||||
require('match-sorter');
|
||||
require('lodash/fp/throttle');
|
||||
var react = require('@mdx-js/react');
|
||||
var createHashSource = require('hash-source');
|
||||
let source = createHashSource.default();;
|
||||
|
||||
const BasePlayground = loadable(() => Promise.resolve(require('./Playground.js')));
|
||||
const Playground = props => typeof window !== 'undefined' ? React.createElement(React.Suspense, {
|
||||
fallback: null
|
||||
}, React.createElement(BasePlayground, Object.assign({}, props))) : null;
|
||||
|
||||
const AsyncComponent = defaultProps => {
|
||||
const [loading, setLoading] = React.useState(true);
|
||||
const [error, setError] = React.useState(null);
|
||||
const [data, setData] = React.useState({});
|
||||
React.useEffect(() => {
|
||||
const {
|
||||
getInitialProps
|
||||
} = defaultProps;
|
||||
|
||||
if (getInitialProps && __chunk_1.isFn(getInitialProps)) {
|
||||
setLoading(true);
|
||||
getInitialProps(defaultProps).then(data => {
|
||||
setLoading(false);
|
||||
setError(null);
|
||||
setData(data);
|
||||
}).catch(err => {
|
||||
setLoading(false);
|
||||
setError(err);
|
||||
setData({});
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
const {
|
||||
as: Comp,
|
||||
getInitialProps
|
||||
} = defaultProps,
|
||||
props = tslib_1.__rest(defaultProps, ["as", "getInitialProps"]);
|
||||
|
||||
return React.createElement(Comp, Object.assign({}, props, {
|
||||
data: Object.assign({}, data, {
|
||||
loading,
|
||||
error
|
||||
})
|
||||
}));
|
||||
};
|
||||
|
||||
const loadRoute = (path, imports, components) => {
|
||||
const Loading = components.loading;
|
||||
|
||||
const fn = async () => {
|
||||
const importFn = _get(path, imports);
|
||||
|
||||
const {
|
||||
default: Component,
|
||||
getInitialProps
|
||||
} = await importFn();
|
||||
|
||||
const ExportedComponent = props => React.createElement(AsyncComponent, Object.assign({}, props, {
|
||||
as: Component || 'div',
|
||||
getInitialProps: getInitialProps
|
||||
}));
|
||||
|
||||
return ExportedComponent;
|
||||
};
|
||||
|
||||
return loadable(fn, {
|
||||
fallback: React.createElement(Loading, null)
|
||||
});
|
||||
};
|
||||
const AsyncRoute = defaultProps => {
|
||||
const {
|
||||
asyncComponent,
|
||||
path,
|
||||
entry
|
||||
} = defaultProps,
|
||||
routeProps = tslib_1.__rest(defaultProps, ["asyncComponent", "path", "entry"]);
|
||||
|
||||
const components = __chunk_1.useComponents();
|
||||
const Page = components.page;
|
||||
const Component = asyncComponent;
|
||||
const props = Object.assign({}, routeProps, {
|
||||
doc: entry
|
||||
});
|
||||
return Page ? React.createElement(Page, Object.assign({}, props), React.createElement(Component, Object.assign({}, props))) : React.createElement(Component, Object.assign({}, props));
|
||||
};
|
||||
|
||||
const Link = React.forwardRef((defaultProps, ref) => {
|
||||
const props = _omit(['activeClassName', 'partiallyActive'], defaultProps);
|
||||
|
||||
const isActive = React.useCallback(({
|
||||
isCurrent
|
||||
}) => {
|
||||
return isCurrent ? {
|
||||
className: `${props.className} active`
|
||||
} : {};
|
||||
}, [props.className]);
|
||||
return React.createElement(router.Link, Object.assign({}, props, {
|
||||
getProps: isActive,
|
||||
ref: ref
|
||||
}));
|
||||
});
|
||||
Link.displayName = 'Link';
|
||||
|
||||
const RE_OBJECTOF = /(?:React\.)?(?:PropTypes\.)?objectOf\((?:React\.)?(?:PropTypes\.)?(\w+)\)/;
|
||||
|
||||
const getTypeStr = type => {
|
||||
switch (type.name.toLowerCase()) {
|
||||
case 'instanceof':
|
||||
return `Class(${type.value})`;
|
||||
|
||||
case 'enum':
|
||||
if (type.computed) return type.value;
|
||||
return type.value ? type.value.map(v => `${v.value}`).join(' │ ') : type.raw;
|
||||
|
||||
case 'union':
|
||||
return type.value ? type.value.map(t => `${getTypeStr(t)}`).join(' │ ') : type.raw;
|
||||
|
||||
case 'array':
|
||||
return type.raw;
|
||||
|
||||
case 'arrayof':
|
||||
return `Array<${getTypeStr(type.value)}>`;
|
||||
|
||||
case 'custom':
|
||||
if (type.raw.indexOf('function') !== -1 || type.raw.indexOf('=>') !== -1) return 'Custom(Function)';else if (type.raw.toLowerCase().indexOf('objectof') !== -1) {
|
||||
const m = type.raw.match(RE_OBJECTOF);
|
||||
if (m && m[1]) return `ObjectOf(${capitalize(m[1])})`;
|
||||
return 'ObjectOf';
|
||||
}
|
||||
return 'Custom';
|
||||
|
||||
case 'bool':
|
||||
return 'Boolean';
|
||||
|
||||
case 'func':
|
||||
return 'Function';
|
||||
|
||||
case 'shape':
|
||||
const shape = type.value;
|
||||
const rst = {};
|
||||
Object.keys(shape).forEach(key => {
|
||||
rst[key] = getTypeStr(shape[key]);
|
||||
});
|
||||
return JSON.stringify(rst, null, 2);
|
||||
|
||||
default:
|
||||
return capitalize(type.name);
|
||||
}
|
||||
};
|
||||
|
||||
const humanize = type => getTypeStr(type);
|
||||
|
||||
const getPropType = prop => {
|
||||
const propName = _get('name', prop.flowType || prop.type);
|
||||
|
||||
if (!propName) return null;
|
||||
const isEnum = propName.startsWith('"') || propName === 'enum';
|
||||
const name = capitalize(isEnum ? 'enum' : propName);
|
||||
|
||||
const value = _get('type.value', prop);
|
||||
|
||||
if (!name) return null;
|
||||
|
||||
if (isEnum && typeof value === 'string' || !prop.flowType && !isEnum && !value || prop.flowType && !prop.flowType.elements) {
|
||||
return name;
|
||||
}
|
||||
|
||||
return prop.flowType ? humanize(prop.flowType) : humanize(prop.type);
|
||||
};
|
||||
const Props = ({
|
||||
title,
|
||||
isToggle,
|
||||
isRaw,
|
||||
of: component
|
||||
}) => {
|
||||
const components = __chunk_1.useComponents();
|
||||
const {
|
||||
props: stateProps
|
||||
} = React.useContext(__chunk_1.doczState.context);
|
||||
const PropsComponent = components.props;
|
||||
|
||||
const filename = _get('__filemeta.filename', component);
|
||||
|
||||
const filemetaName = _get('__filemeta.name', component);
|
||||
|
||||
const componentName = filemetaName || component.displayName || component.name;
|
||||
const found = stateProps && stateProps.length > 0 && stateProps.find(item => filename ? item.key === filename : item.key.includes(`${componentName}.`));
|
||||
const value = _get('value', found) || [];
|
||||
|
||||
const firstDefinition = _first(value);
|
||||
|
||||
const definition = value.find(i => i.displayName === componentName);
|
||||
|
||||
const props = _get('props', definition || firstDefinition);
|
||||
|
||||
if (!props) return null;
|
||||
if (!PropsComponent) return null;
|
||||
return React.createElement(PropsComponent, {
|
||||
title: title,
|
||||
isRaw: isRaw,
|
||||
isToggle: isToggle,
|
||||
props: props,
|
||||
getPropType: getPropType
|
||||
});
|
||||
};
|
||||
|
||||
const goToHash = ({
|
||||
location
|
||||
}) => {
|
||||
setTimeout(() => {
|
||||
if (location && location.hash) {
|
||||
const decodedHash = decodeURI(location.hash);
|
||||
const id = decodedHash.substring(1);
|
||||
const el = document.getElementById(id);
|
||||
if (el) el.scrollIntoView();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const Routes = ({
|
||||
imports
|
||||
}) => {
|
||||
const components = __chunk_1.useComponents();
|
||||
const {
|
||||
entries
|
||||
} = React.useContext(__chunk_1.doczState.context);
|
||||
const NotFound = components.notFound;
|
||||
const history = React.useMemo(() => router.createHistory(source), []);
|
||||
React.useEffect(() => {
|
||||
history.listen(goToHash);
|
||||
}, []);
|
||||
return React.createElement(react.MDXProvider, {
|
||||
components: components
|
||||
}, React.createElement(router.LocationProvider, {
|
||||
history: history
|
||||
}, React.createElement(router.Router, null, React.createElement(NotFound, {
|
||||
default: true
|
||||
}), entries && entries.map(({
|
||||
key: path,
|
||||
value: entry
|
||||
}) => {
|
||||
const props = {
|
||||
path,
|
||||
entries,
|
||||
components
|
||||
};
|
||||
const component = loadRoute(path, imports, components);
|
||||
return React.createElement(AsyncRoute, Object.assign({}, props, {
|
||||
entry: entry,
|
||||
key: entry.id,
|
||||
path: entry.route,
|
||||
asyncComponent: component
|
||||
}));
|
||||
}))));
|
||||
};
|
||||
|
||||
function theme(themeConfig, transform = c => c) {
|
||||
return WrappedComponent => {
|
||||
const Theme = React.memo(props => {
|
||||
const {
|
||||
linkComponent
|
||||
} = props;
|
||||
const {
|
||||
db,
|
||||
children,
|
||||
wrapper: Wrapper = React.Fragment
|
||||
} = props;
|
||||
const initial = Object.assign({}, db, {
|
||||
themeConfig,
|
||||
transform,
|
||||
linkComponent
|
||||
});
|
||||
return React.createElement(__chunk_1.doczState.Provider, {
|
||||
initial: initial
|
||||
}, React.createElement(Wrapper, null, React.createElement(WrappedComponent, null, children)));
|
||||
});
|
||||
Theme.displayName = WrappedComponent.displayName || 'DoczTheme';
|
||||
return Theme;
|
||||
};
|
||||
}
|
||||
|
||||
exports.ComponentsProvider = __chunk_1.ComponentsProvider;
|
||||
exports.doczState = __chunk_1.doczState;
|
||||
exports.useComponents = __chunk_1.useComponents;
|
||||
exports.useConfig = __chunk_1.useConfig;
|
||||
exports.useDataServer = __chunk_1.useDataServer;
|
||||
exports.useDocs = __chunk_1.useDocs;
|
||||
exports.useMenus = __chunk_1.useMenus;
|
||||
exports.usePrevious = __chunk_1.usePrevious;
|
||||
exports.useWindowSize = __chunk_1.useWindowSize;
|
||||
exports.AsyncRoute = AsyncRoute;
|
||||
exports.Link = Link;
|
||||
exports.Playground = Playground;
|
||||
exports.Props = Props;
|
||||
exports.Routes = Routes;
|
||||
exports.loadRoute = loadRoute;
|
||||
exports.theme = theme;
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import { ComponentType } from 'react';
|
||||
export interface Heading {
|
||||
depth: number;
|
||||
slug: string;
|
||||
value: string;
|
||||
}
|
||||
export interface Entry {
|
||||
id: string;
|
||||
filepath: string;
|
||||
slug: string;
|
||||
route: string;
|
||||
name: string;
|
||||
order: number;
|
||||
menu: string | null;
|
||||
headings: Heading[];
|
||||
[key: string]: any;
|
||||
}
|
||||
export interface MenuItem {
|
||||
id: string;
|
||||
name: string;
|
||||
route?: string;
|
||||
href?: string;
|
||||
menu?: MenuItem[];
|
||||
order?: number;
|
||||
parent?: string;
|
||||
}
|
||||
export declare type ThemeConfig = Record<string, any>;
|
||||
export interface Config {
|
||||
title: string;
|
||||
description: string;
|
||||
themeConfig: ThemeConfig;
|
||||
menu: MenuItem[];
|
||||
version: string | null;
|
||||
repository: string | null;
|
||||
native: boolean;
|
||||
separator: string;
|
||||
codeSandbox: boolean;
|
||||
base?: string;
|
||||
}
|
||||
export declare type Entries = {
|
||||
key: string;
|
||||
value: Entry;
|
||||
}[];
|
||||
export declare type Props = {
|
||||
key: string;
|
||||
value: any;
|
||||
}[];
|
||||
export declare type TransformFn = (config: ThemeConfig) => ThemeConfig;
|
||||
export interface Database {
|
||||
config: Config;
|
||||
props?: Props;
|
||||
entries?: Entries;
|
||||
}
|
||||
export interface DoczState extends Database {
|
||||
linkComponent: ComponentType<any>;
|
||||
themeConfig?: ThemeConfig;
|
||||
transform?: TransformFn;
|
||||
}
|
||||
export declare const doczState: import("./utils/createState").State<DoczState>;
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
import { ComponentType as CT } from 'react';
|
||||
import { Database, ThemeConfig, TransformFn } from './state';
|
||||
export interface ThemeProps {
|
||||
db: Database;
|
||||
wrapper?: CT;
|
||||
linkComponent?: CT;
|
||||
children(WrappedComponent: CT): JSX.Element;
|
||||
}
|
||||
export declare type ThemeReturn = (WrappedComponent: CT) => CT<ThemeProps>;
|
||||
export declare function theme(themeConfig: ThemeConfig, transform?: TransformFn): ThemeReturn;
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import * as React from 'react';
|
||||
export interface ProviderProps<T> {
|
||||
initial?: T;
|
||||
}
|
||||
export declare type PrevState<T> = (prevState: T) => T;
|
||||
export declare type GetFn<T> = (state: T) => React.ReactNode;
|
||||
export declare type Dispatch<T> = T | PrevState<T>;
|
||||
export interface State<T> {
|
||||
context: React.Context<T>;
|
||||
set: (param: Dispatch<T>) => void;
|
||||
Provider: React.ComponentType<ProviderProps<T>>;
|
||||
}
|
||||
export declare function create<T = any>(initial: T): State<T>;
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
export declare const isFn: (value: any) => boolean;
|
||||
export declare function flatArrFromObject<T>(arr: T[], prop: string): string[];
|
||||
export declare function compare<T>(a: T, b: T, reverse?: boolean): number;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
import { PropType, FlowType } from '../components/Props';
|
||||
export declare const humanize: (type: PropType | FlowType) => any;
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import menu from "./docz-lib/config/menu";
|
||||
import versions from './docz-lib/config/versions';
|
||||
|
||||
module.exports = {
|
||||
title: "Apache Atlas",
|
||||
files: "**/*.{md,mdx}",
|
||||
base: "/",
|
||||
baseUrl:"./public",
|
||||
src: "./src",
|
||||
public: "./src/resources",
|
||||
menu: menu,
|
||||
atlasVersions: versions,
|
||||
theme: "theme/",
|
||||
modifyBundlerConfig: config => {
|
||||
config.module.rules.push(
|
||||
{
|
||||
test: /\.(js)$/,
|
||||
exclude: /node_modules/,
|
||||
use: {
|
||||
loader: "babel-loader",
|
||||
query: {
|
||||
presets: ["@babel/react"],
|
||||
plugins: [
|
||||
"@babel/plugin-proposal-class-properties",
|
||||
"@babel/plugin-syntax-dynamic-import"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
return config;
|
||||
}
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,39 @@
|
|||
{
|
||||
"name": "doc",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node ./docz-lib/docz/bin/index.js dev",
|
||||
"build": "node ./docz-lib/docz/bin/index.js build",
|
||||
"predeploy": "npm install && npm run build",
|
||||
"deploy": "gh-pages -d .docz/dist"
|
||||
},
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.4.4",
|
||||
"@babel/core": "^7.4.5",
|
||||
"@babel/plugin-syntax-dynamic-import": "^7.2.0",
|
||||
"@babel/preset-react": "^7.0.0",
|
||||
"axios": "^0.19.0",
|
||||
"babel-loader": "^8.0.6",
|
||||
"babel-plugin-lodash": "^3.3.4",
|
||||
"babel-plugin-react-transform": "^3.0.0",
|
||||
"gh-pages": "^2.0.1",
|
||||
"react-dropdown-select": "^3.0.0",
|
||||
"react-github-btn": "^1.0.5",
|
||||
"react-syntax-highlighter": "^11.0.1",
|
||||
"reactstrap": "^8.0.0",
|
||||
"xml2js": "^0.4.19"
|
||||
},
|
||||
"dependencies": {
|
||||
"docz": "^1.2.0",
|
||||
"facepaint": "^1.2.1",
|
||||
"hash-source": "^1.0.4",
|
||||
"react-feather": "^1.1.6",
|
||||
"styled-components": "^4.3.1",
|
||||
"yargs": "^14.0.0"
|
||||
}
|
||||
}
|
||||
104
docs/pom.xml
104
docs/pom.xml
|
|
@ -1,5 +1,4 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one
|
||||
~ or more contributor license agreements. See the NOTICE file
|
||||
|
|
@ -27,13 +26,12 @@
|
|||
<artifactId>atlas-docs</artifactId>
|
||||
<description>Apache Atlas Documentation</description>
|
||||
<name>Apache Atlas Documentation</name>
|
||||
|
||||
<properties>
|
||||
<skipTests>true</skipTests>
|
||||
<skipSite>false</skipSite>
|
||||
</properties>
|
||||
|
||||
<reporting>
|
||||
<!-- TODO -->
|
||||
<!-- <reporting>
|
||||
<excludeDefaults>true</excludeDefaults>
|
||||
<outputDirectory>${project.build.directory}/site</outputDirectory>
|
||||
<plugins>
|
||||
|
|
@ -63,47 +61,81 @@
|
|||
</reportSets>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</reporting>
|
||||
|
||||
</reporting> -->
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.github.eirslett</groupId>
|
||||
<artifactId>frontend-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<phase>validate</phase>
|
||||
<workingDirectory>${project.build.directory}</workingDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-site-plugin</artifactId>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven.doxia</groupId>
|
||||
<artifactId>doxia-module-twiki</artifactId>
|
||||
<version>${doxia.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven.doxia</groupId>
|
||||
<artifactId>doxia-module-markdown</artifactId>
|
||||
<version>${doxia.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven.doxia</groupId>
|
||||
<artifactId>doxia-module-markdown</artifactId>
|
||||
<version>${doxia.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven.doxia</groupId>
|
||||
<artifactId>doxia-core</artifactId>
|
||||
<version>${doxia.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<configuration>
|
||||
<port>8888</port>
|
||||
</configuration>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
<version>2.7</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-resources</id>
|
||||
<goals>
|
||||
<goal>site</goal>
|
||||
<goal>copy-resources</goal>
|
||||
</goals>
|
||||
<phase>prepare-package</phase>
|
||||
<phase>validate</phase>
|
||||
<configuration>
|
||||
<outputDirectory>${basedir}/target</outputDirectory>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>${basedir}</directory>
|
||||
<excludes>
|
||||
<exclude>.docz/**</exclude>
|
||||
<exclude>node_modules/**</exclude>
|
||||
<exclude>target/**</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
</resources>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>com.github.eirslett</groupId>
|
||||
<artifactId>frontend-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>install node and npm</id>
|
||||
<goals>
|
||||
<goal>install-node-and-npm</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<nodeVersion>${node-for-v2.version}</nodeVersion>
|
||||
<npmVersion>${npm-for-v2.version}</npmVersion>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>npm install</id>
|
||||
<goals>
|
||||
<goal>npm</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<arguments>install</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>npm build</id>
|
||||
<goals>
|
||||
<goal>npm</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<arguments>run build</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
name: ASF
|
||||
route: /asf
|
||||
menu: ASF
|
||||
---
|
||||
|
||||
import {CustomLink} from "theme/components/shared/common/CustomLink";
|
||||
|
||||
# ASF Infomation
|
||||
|
||||
|
||||
1. <CustomLink href="http://www.apache.org/foundation/how-it-works.html">How Apache Works</CustomLink>
|
||||
2. <CustomLink href="https://apachecon.com/?ref=atlas.apache.org">Events </CustomLink>
|
||||
3. <CustomLink href="https://www.apache.org/licenses/">License </CustomLink>
|
||||
4. <CustomLink href="http://www.apache.org/foundation/">Foundation </CustomLink>
|
||||
5. <CustomLink href="http://www.apache.org/foundation/sponsorship.html">Sponsoring Apache </CustomLink>
|
||||
6. <CustomLink href="http://www.apache.org/foundation/thanks.html">Thanks</CustomLink>
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
---
|
||||
name: Classification Propagation
|
||||
route: /ClassificationPropagation
|
||||
menu: Documentation
|
||||
submenu: Features
|
||||
---
|
||||
import Img from 'theme/components/shared/Img'
|
||||
|
||||
# Classification Propagation
|
||||
|
||||
* Classification propagation enables classifications associated to an entity to be automatically associated with other related entities of the entity. This is very useful in dealing with scenarios where a dataset derives it data from other datasets - like a table loaded with data in a file, a report generated from a table/view, etc.
|
||||
* For example, when a table is classified as *PII*, tables or views that derive data from this table (via CTAS or ‘create view’ operation) will be automatically classified as *PII*.
|
||||
|
||||
|
||||
## Use Cases
|
||||
|
||||
Consider the following lineage where data from a 'hdfs_path' entity is loaded into a table, which is further made available through views. We will go through various scenarios to understand the classification propagation feature.
|
||||
<Img src={`/images/twiki/classification-propagation-1.png`}/>
|
||||
|
||||
|
||||
## Add classification to an entity
|
||||
|
||||
When classification ‘PII’ is added to 'hdfs_path' entity, the classification is propagated to all impacted entities in the lineage path, including 'employees' table, views 'us_employees' and 'uk_employees' - as shown below.
|
||||
<Img src={`/images/twiki/classification-propagation-2.png`}/>
|
||||
|
||||
## Update classification associated with an entity
|
||||
|
||||
Any updates to classifications associated with an entity will be seen in all entities the classification is propagated to as well.
|
||||
<Img src={`/images/twiki/classification-propagation-3.png`}/>
|
||||
|
||||
## Remove classification associated with an entity
|
||||
|
||||
When a classification is deleted from an entity, the classification will be removed from all entities the classification is propagated to as well.
|
||||
<Img src={`/images/twiki/classification-propagation-4.png`}/>
|
||||
|
||||
## Add lineage between entities
|
||||
|
||||
When lineage is added between entities, for example to capture loading of data in a file to a table, the classifications associated with the source entity are propagated to all impacted entities as well.
|
||||
For example, when a view is created from a table, classifications associated with the table are propagated to the newly created view as well.
|
||||
<Img src={`/images/twiki/classification-propagation-5.png`}/>
|
||||
|
||||
## Delete an entity
|
||||
|
||||
**Case 1:**
|
||||
When an entity is deleted, classifications associated with this entity will be removed from all entities the classifications are propagated to.
|
||||
For example. when _employees_ table is deleted, classifications associated with this table are removed from 'employees_view' view.
|
||||
<Img src={`/images/twiki/classification-propagation-6.png`}/>
|
||||
|
||||
**Case 2:**
|
||||
When an entity is deleted in the middle of a lineage path, the propagation link is broken and previously propagated classifications will be removed from all derived entities of the deleted entity.
|
||||
For example. when 'us_employees' table is deleted, classifications propagating through this table (**PII**) are removed from 'ca_employees' table, since the only path of propagation is broken by entity deletion.
|
||||
<Img src={`/images/twiki/classification-propagation-entity-delete-1.png"`}/>
|
||||
|
||||
<Img src={`/images/twiki/classification-propagation-entity-delete-2.png"`}/>
|
||||
|
||||
**Case 3:**
|
||||
When an entity is deleted in the middle of a lineage path and if there exists alternate path for propagation, previously propagated classifications will be retained.
|
||||
For example. when 'us_employees' table is deleted, classifications propagating (**PII**) through this table are retained in 'ca_employees' table, since there are two propagation paths available and only one of them is broken by entity deletion.
|
||||
<Img src={`/images/twiki/classification-propagation-entity-delete-3.png"`}/>
|
||||
|
||||
<Img src={`/images/twiki/classification-propagation-entity-delete-4.png"`}/>
|
||||
|
||||
## Control Propagation
|
||||
|
||||
Apache Atlas provides few options to control whether/where a classification is propagated.
|
||||
This section details available options.
|
||||
|
||||
## Propagate flag in classification
|
||||
|
||||
Each association of classification to an entity has a boolean flag that controls whether the classification is propagated or not.
|
||||
When a classification is associated with an entity, this flag is set to ‘true’ i.e. the classification will be propagated to all impacted entities. This flag can be updated to desired value during initial association or later.
|
||||
<Img src={`/images/twiki/classification-propagation-7.png`}/>
|
||||
|
||||
## Propagate flag in lineage edge
|
||||
|
||||
Apache Atlas supports a flag at lineage edge to enable/disable propagation of classifications through the edge. By default, the propagation is enabled for lineage edges.
|
||||
When the flag is turned off, no classification will be propagated through this edge; and propagation of currently propagated classifications through the edge will be reevaluated, so that they can be removed from impacted entities.
|
||||
When the flag is turned on, propagation of classifications of the source entity will be reevaluated, so that they can be propagated to all impacted entities.
|
||||
|
||||
## Block propagation of specific classifications in lineage edge
|
||||
|
||||
Apache Atlas supports blocking propagation of specific classifications in at lineage edges.
|
||||
This can be useful, for example, to handle scenarios like: a column classified as PII is masked when creating a view; in such scenario, if corresponding column in the created view might not have PII, hence the propagation of PII classification should be blocked.
|
||||
This can be done by updating the lineage edge to add the PII classification in _‘blocked propagated classifications’_ list.
|
||||
Classifications in blocked propagated classifications will not be propagated in the derivative/downstream entities.
|
||||
<Img src={`/images/twiki/classification-propagation-8.png`}/>
|
||||
|
||||
## Notifications and Audit
|
||||
|
||||
When propagated classifications are added/update/deleted, Apache Atlas sends notifications to 'ATLAS_ENTITIES' topic for each entity affected by the propagation.
|
||||
|
||||
## Glossary
|
||||
|
||||
When a classification is associated with a glossary term, the classification is automatically propagated to all entities associated with the term.
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
---
|
||||
name: Download
|
||||
route: /Downloads
|
||||
menu: Downloads
|
||||
submenu: Download
|
||||
---
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Downloads
|
||||
|
||||
|
||||
Apache Atlas release artifacts are distributed via mirror sites and should be checked for tampering using GPG or SHA-256.
|
||||
|
||||
The table below lists release artifacts and their associated signatures and hashes. The keys used to sign the release
|
||||
artifacts can be found in our published [KEYS file](https://www.apache.org/dist/atlas/KEYS).
|
||||
|
||||
| **Version** | **Release Date** | **Tarball** | **GPG** | **Hash** |
|
||||
| : ------------- : | : ------------- : | : ------------- : | : ------------- : |: ------------- :|
|
||||
| 2.0.0 | 2019-05-14 | [source](https://www.apache.org/dyn/closer.cgi/atlas/2.0.0/apache-atlas-2.0.0-sources.tar.gz) | [signature](https://www.apache.org/dist/atlas/2.0.0/apache-atlas-2.0.0-sources.tar.gz.asc) | [SHA512](https://www.apache.org/dist/atlas/2.0.0/apache-atlas-2.0.0-sources.tar.gz.sha512) |
|
||||
| 1.1.0 | 2018-09-17 | [source](https://www.apache.org/dyn/closer.cgi/atlas/1.1.0/apache-atlas-1.1.0-sources.tar.gz) | [signature](https://www.apache.org/dist/atlas/1.1.0/apache-atlas-1.1.0-sources.tar.gz.asc) | [SHA512](https://www.apache.org/dist/atlas/1.1.0/apache-atlas-1.1.0-sources.tar.gz.sha512) |
|
||||
| 1.0.0 | 2018-06-02 | [source](https://www.apache.org/dyn/closer.cgi/atlas/1.0.0/apache-atlas-1.0.0-sources.tar.gz) | [signature](https://www.apache.org/dist/atlas/1.0.0/apache-atlas-1.0.0-sources.tar.gz.asc) | [SHA512](https://www.apache.org/dist/atlas/1.0.0/apache-atlas-1.0.0-sources.tar.gz.sha512) |
|
||||
| 0.8.2 | 2018-02-05 | [source](https://www.apache.org/dyn/closer.cgi/atlas/0.8.2/apache-atlas-0.8.2-sources.tar.gz) | [signature](https://www.apache.org/dist/atlas/0.8.2/apache-atlas-0.8.2-sources.tar.gz.asc) | [SHA512](https://www.apache.org/dist/atlas/0.8.2/apache-atlas-0.8.2-sources.tar.gz.sha512) |
|
||||
| 0.8.1 | 2017-08-29 | [source](https://archive.apache.org/dist/atlas/0.8.1/apache-atlas-0.8.1-sources.tar.gz) | [signature](https://archive.apache.org/dist/atlas/0.8.1/apache-atlas-0.8.1-sources.tar.gz.asc) | [SHA512](https://archive.apache.org/dist/atlas/0.8.1/apache-atlas-0.8.1-sources.tar.gz.sha512) |
|
||||
| 0.8.0-incubating | 2017-03-16 | [source](https://archive.apache.org/dist/incubator/atlas/0.8.0-incubating/apache-atlas-0.8-incubating-sources.tar.gz) | [signature](https://archive.apache.org/dist/incubator/atlas/0.8.0-incubating/apache-atlas-0.8-incubating-sources.tar.gz.asc) | [SHA512](https://archive.apache.org/dist/incubator/atlas/0.8.0-incubating/apache-atlas-0.8-incubating-sources.tar.gz.sha512) |
|
||||
| 0.7.1-incubating | 2017-01-29 | [source](https://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/apache-atlas-0.7.1-incubating-sources.tar.gz) | [signature](https://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/apache-atlas-0.7.1-incubating-sources.tar.gz.asc) | [checksum](https://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/apache-atlas-0.7.1-incubating-sources.tar.gz.mds) |
|
||||
| 0.7.0-incubating | 2016-07-09 | [source](https://archive.apache.org/dist/incubator/atlas/0.7.0-incubating/apache-atlas-0.7-incubating-sources.tar.gz) | [signature](https://archive.apache.org/dist/incubator/atlas/0.7.0-incubating/apache-atlas-0.7-incubating-sources.tar.gz.asc) | [SHA512](https://archive.apache.org/dist/incubator/atlas/0.7.0-incubating/apache-atlas-0.7-incubating-sources.tar.gz.sha512) |
|
||||
| 0.6.0-incubating | 2015-12-31 | [source](https://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/apache-atlas-0.6-incubating-sources.tar.gz) | [signature](https://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/apache-atlas-0.6-incubating-sources.tar.gz.asc) | [SHA](https://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/apache-atlas-0.6-incubating-sources.tar.gz.sha) |
|
||||
| 0.5.0-incubating | 2015-06-22 | [source](https://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/apache-atlas-0.5-incubating-sources.tar.gz) | [signature](https://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/apache-atlas-0.5-incubating-sources.tar.gz.asc) | [SHA](https://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/apache-atlas-0.5-incubating-sources.tar.gz.sha) |
|
||||
|
||||
## Verify the integrity of the files
|
||||
|
||||
It is essential that you verify the integrity of the downloaded file using the PGP signature (.asc file) or a hash
|
||||
(.md5 or .sha* file). Please read [Verifying Apache Software Foundation Releases](https://www.apache.org/info/verification.html)
|
||||
for more information on why you should verify our releases.
|
||||
|
||||
The PGP signature can be verified using PGP or GPG, with the following steps:
|
||||
* Download the release artifact from the link in the table above
|
||||
* Download the signature file for the release from the link in the table above
|
||||
* Download [Apache Atlas KEYS file](https://www.apache.org/dist/atlas/KEYS)
|
||||
* Verify the signatures using one of the following:
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
% gpg --import KEYS
|
||||
% gpg --verify downloaded_file.asc downloaded_file
|
||||
</SyntaxHighlighter>
|
||||
|
||||
or
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
% pgpk -a KEYS
|
||||
% pgpv downloaded_file.asc
|
||||
</SyntaxHighlighter>
|
||||
|
||||
or
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
% pgp -ka KEYS
|
||||
% pgp downloaded_file.asc
|
||||
</SyntaxHighlighter>
|
||||
|
||||
## Release Notes
|
||||
**[Atlas 2.0.0](../2.0.0/index) (Released on 2019/05/14)**
|
||||
* Soft-reference attribute implementation.
|
||||
* Unique-attributes constraints at graph store-level
|
||||
* Atlas Index Repair tool for Janusgraph
|
||||
* Relationship notifications when new relationships are created in atlas
|
||||
* Atlas Import Transform handler implementation
|
||||
* Updated component versions to use Hadoop 3.1, Hive 3.1, HBase 2.0, Solr 7.5 and Kafka 2.0
|
||||
* Updated JanusGraph version to 0.3.1
|
||||
* Updated authentication to support trusted proxy
|
||||
* Updated patch framework to persist typedef patches applied to atlas and handle data patches.
|
||||
* Updated metrics module to collect notification metrics
|
||||
* Updated Atlas Export to support incremental export of metadata.
|
||||
* Notification Processing Improvements:
|
||||
* Notification processing to support batch-commits
|
||||
* New option in notification processing to ignore potentially incorrect hive_column_lineage
|
||||
* Updated Hive hook to avoid duplicate column-lineage entities; also updated Atlas server to skip duplicate column-lineage entities
|
||||
* Improved batch processing in notificaiton handler to avoid processing of an entity multiple times
|
||||
* Add option to ignore/prune metadata for temporary/staging hive tables
|
||||
* Avoid unnecessary lookup when creating new relationships
|
||||
* UI Improvements:
|
||||
* UI: Display counts besides the Type and Classification dropdown list in basic search
|
||||
* UI: Display lineage information for process entities
|
||||
* UI: Display entity specific icon for the lineage graph
|
||||
* UI: Add relationships table inside relationships view in entity details page.
|
||||
* UI: Add service-type dropdown in basic search to filter entitydef type.
|
||||
* Various Bug-fixes and optimizations
|
||||
* [List of JIRAs resolved in Apache Atlas 2.0.0 release](https://issues.apache.org/jira/issues/?jql=project%20%3D%20ATLAS%20AND%20status%20%3D%20Resolved%20AND%20fixVersion%20%3D%202.0.0%20ORDER%20BY%20updated%20DESC%2C%20priority%20DESC)
|
||||
|
||||
**[Atlas 1.1.0](../1.1.0/index) (Released on 2018/09/17)**
|
||||
* Updated authorization model to support access control on relationship operations
|
||||
* Added support for AWS S3 datatypes, in Atlas server and Hive hook
|
||||
* Updated [[http://atlas.apache.org/JanusGraph.html][JanusGraph]] version from 0.2.0 to 0.3.0
|
||||
* Updated hooks to send Kafka notifications asynchronously
|
||||
* Enhanced classification-propagation with options to handle entity-deletes
|
||||
* BugFixes and Optimizations
|
||||
|
||||
**[Atlas 1.0.0](../1.0.0/index) (Released on 2018/06/02)**
|
||||
|
||||
* Core model enhancement to support Relationship as first-class construct
|
||||
* Support for JanusGraph graph database
|
||||
* New DSL implementation, using ANTLR instead of Scala
|
||||
* Removal of older type system implementation in atlas-typesystem library
|
||||
* Metadata security - fine grained authorization
|
||||
* Notification enhancements to support V2 style data structures
|
||||
* Jackson library update from 1.9.13 to 2.9.2
|
||||
* Classification propagation via entity relationships
|
||||
* Glossary terms, categories
|
||||
* HBase Hook
|
||||
* UI updates to show entity relationships
|
||||
* [List of JIRAs resolved in Apache Atlas 1.0.0 release](https://issues.apache.org/jira/issues/?jql=project%20%3D%20Atlas%20AND%20resolution%20%3D%20Fixed%20AND%20fixVersion%20%3D%201.0.0%20ORDER%20BY%20key%20DESC)
|
||||
|
||||
**[Atlas 0.8.2](../0.8.2/index) (Released on 2018/02/05)**
|
||||
|
||||
* Search improvements:
|
||||
* Basic search enhancement to optionally exclude sub-type entities and sub-classification-types
|
||||
* Basic search to return classification attributes
|
||||
* Support for saving searches
|
||||
* UI support to reorder columns in search results page
|
||||
* UI - updates for classification rendering – tree/flat view
|
||||
* UI – minification of js, css; cache busting for static content (css, js)
|
||||
* notification updates to handle large messages
|
||||
* fix type initialization issues in HA deployment
|
||||
* In HA, the passive node redirects the request with wrong URL encoding
|
||||
* tool kit to recover from lost/bad index data
|
||||
* [List of JIRAs resolved in Apache Atlas 0.8.2 release](https://issues.apache.org/jira/issues/?jql=project%20%3D%20Atlas%20AND%20fixVersion%20%3D%200.8.2%20ORDER%20BY%20key%20ASC)
|
||||
|
||||
**[Atlas 0.8.1](../0.8.1/index) (Released on 2017/08/29)**
|
||||
|
||||
* Basic-search improvement in use of index for attribute filtering
|
||||
* DSL query enhancement to support 'like' operator
|
||||
* REST API and UI enhancements to update classification attributes
|
||||
* Export/import support to copy data between Apache Atlas instances
|
||||
* Ability to delete a tag from UI (and API)
|
||||
* UI enhancements: lineage, attribute search filter, ability to search historical data
|
||||
* Knox SSO for Atlas REST APIs
|
||||
* Moved from use of Guice + Spring framework for dependency-injection to use only Spring framework
|
||||
|
||||
**[Atlas 0.8-incubating](../0.8.0-incubating/index) (Released on 2017/03/16)**
|
||||
|
||||
* API revamp - new, structured REST API
|
||||
* Simplified search UI
|
||||
* UI to create/update entities - HDFS/HBase/Kafka
|
||||
* Performance and scalability improvements
|
||||
* Knox SSO for Atlas UI
|
||||
|
||||
**[Atlas 0.7.1-incubating](../0.7.1-incubating/index) (Released on 2017/01/29)**
|
||||
|
||||
* Performance and scalability improvements (ATLAS-1403, ATLAS-1404)
|
||||
* Bug fixes
|
||||
|
||||
**[Atlas 0.7-incubating](../0.7.0-incubating/index) (Released on 2016/07/09)**
|
||||
|
||||
* High Availability (ATLAS-510)
|
||||
* Business Catalog / Taxonomy (ATLAS-491)
|
||||
* Improved Hive Integration (ATLAS-492)
|
||||
* Improved Sqoop Integration
|
||||
* Improved Falcon Integration
|
||||
* Improved Storm Integration
|
||||
* Improved Ambari Deployment & Management Support
|
||||
* Entity Versioning / Audit (ATLAS-493)
|
||||
* Performance Improvements
|
||||
* Authorization (ATLAS-497)
|
||||
* Atlas / Ranger Authorization Integration (ATLAS-495)
|
||||
* Standalone HBase Support (ATLAS-498)
|
||||
* Upgrade Support (ATLAS-631)
|
||||
|
||||
**[Atlas 0.6-incubating](../0.6.0-incubating/index) (Released on 2015/12/30)**
|
||||
* Improved Hive Server 2 Integration
|
||||
* Sqoop Integration
|
||||
* Falcon Integration
|
||||
* Storm Integration
|
||||
* Various Bug Fixes
|
||||
* Atlas / Ranger Integration
|
||||
|
||||
**[Atlas 0.5-incubating](../0.5.0-incubating/index) (Released on 2015/07/09)**
|
||||
* Hive Server 2 Integration
|
||||
* Basic Hive Lineage
|
||||
* Basic Ambari Integration
|
||||
* Kerberos Support
|
||||
* DSL for query of datastore
|
||||
* Basic Storage of Metadata
|
||||
* Support for BerkleyDB
|
||||
* Support for Titan 0.5
|
||||
* Support
|
||||
|
|
@ -1,20 +1,14 @@
|
|||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one
|
||||
~ or more contributor license agreements. See the NOTICE file
|
||||
~ distributed with this work for additional information
|
||||
~ regarding copyright ownership. The ASF licenses this file
|
||||
~ to you under the Apache License, Version 2.0 (the
|
||||
~ "License"); you may not use this file except in compliance
|
||||
~ with the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
---
|
||||
name: Glossary
|
||||
route: /Glossary
|
||||
menu: Documentation
|
||||
submenu: Features
|
||||
---
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
import Img from 'theme/components/shared/Img'
|
||||
|
||||
# Glossary
|
||||
|
||||
|
|
@ -23,7 +17,8 @@ other and categorized so that they can be understood in different contexts. Thes
|
|||
like a Database, tables, columns etc. This helps abstract the technical jargon associated with the repositories and
|
||||
allows the user to discover/work with data in the vocabulary that is more familiar to them.
|
||||
|
||||
###Use cases
|
||||
### Use cases
|
||||
|
||||
* Ability to define rich glossary vocabularies using the natural terminology (technical terms and/or business terms).
|
||||
* Ability to semantically relate the term(s) to each other.
|
||||
* Ability to map assets to glossary terms(s).
|
||||
|
|
@ -31,7 +26,7 @@ allows the user to discover/work with data in the vocabulary that is more famili
|
|||
* Allow categories to be arranged in hierarchy - to express broader and finer scopes.
|
||||
* Separate management of glossary term(s) from the metadata.
|
||||
|
||||
###What is a Glossary term ?
|
||||
### What is a Glossary term ?
|
||||
|
||||
A term is a useful word for an enterprise. For the term(s) to be useful and meaningful, they need to grouped around their
|
||||
use and context. A term in Apache Atlas must have a unique qualifiedName, there can be term(s) with same name but they
|
||||
|
|
@ -84,25 +79,25 @@ Category view allows an user to perform the following operations:
|
|||
|
||||
Users can switch between term view and category view using toggle provided in GLOSSARY tab.
|
||||
|
||||
<img src="images/markdown/terms_view.png" width="250" style="border:2px solid black; margin-left:50px"/>
|
||||
<p/>
|
||||
<img src="images/markdown/category_view_1.png" width="250" style="border:2px solid black; margin-left:50px"/>
|
||||
|
||||
<Img src={`/images/markdown/terms_view.png`} width="300px" />
|
||||
|
||||
<Img src={`/images/markdown/category_view_1.png`} width="300px" />
|
||||
|
||||
|
||||
##### Term context menu
|
||||
|
||||
* Create a new term
|
||||
* Create a new term
|
||||
Clicking on the **ellipsis (...)** next to a glossary name shows a pop-over menu that allows users to create a term in
|
||||
the glossary or delete the glossary - as shown below.
|
||||
|
||||
<img src="images/markdown/term_view_context.png" width="400" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/term_view_context.png`} width="360px" />
|
||||
|
||||
|
||||
* To delete a term
|
||||
* To delete a term
|
||||
Clicking on the **ellipsis (...)** next to a term name shows a pop-over menu that allows users to delete the term - as
|
||||
shown below.
|
||||
|
||||
<img src="images/markdown/term_delete_context.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/term_delete_context.png`} width="600px" />
|
||||
|
||||
|
||||
##### Term detail page
|
||||
|
|
@ -113,109 +108,109 @@ page provides different details of the term.
|
|||
* Classifications tab shows the classification(s) associated with the selected term
|
||||
* Related terms tab shows the terms that are related to the selected term
|
||||
|
||||
<img src="images/markdown/term_details.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/term_details.png`} />
|
||||
|
||||
|
||||
##### Add classification to Term
|
||||
|
||||
Clicking on **+** next to classification label to add a classification to the term.
|
||||
|
||||
<img src="images/markdown/term_add_classification_1.png" width="400" style="border:2px solid black; margin-left:50px" alt="Add classification"/>
|
||||
<p/>
|
||||
<img src="images/markdown/term_add_classification_2.png" width="400" style="border:2px solid black; margin-left:50px" alt="Add classification - details"/>
|
||||
<p/>
|
||||
<img src="images/markdown/term_with_classification.png" width="400" style="border:2px solid black; margin-left:50px" alt="Classifications associated with term"/>
|
||||
<Img src={`/images/markdown/term_add_classification_1.png`} alt="Add classification" width="400px" />
|
||||
|
||||
<Img src={`/images/markdown/term_add_classification_2.png`} alt="Add classification - details" width="400px" />
|
||||
|
||||
<Img src={`/images/markdown/term_with_classification.png`} alt="Classifications associated with term" width="400px" />
|
||||
|
||||
|
||||
##### Create term relationship with other term
|
||||
Click on "Related Terms" tab when viewing term details. Clicking on **+** will allow linking a term with the current term.
|
||||
|
||||
<img src="images/markdown/terms_related_terms.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/terms_related_terms.png`} />
|
||||
|
||||
|
||||
##### Categorize a term
|
||||
|
||||
Click on **+** next to categories label to categorize a term. A modal dialog will be presented for choosing a category.
|
||||
|
||||
<img src="images/markdown/term_add_category.png" width="500" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/term_add_category.png`} width="400px" />
|
||||
|
||||
|
||||
#### **Category view**
|
||||
|
||||
When the toggle switch is on category, the panel will list down all glossaries along-with the category hierarchy. Here's a list of possible
|
||||
When the toggle switch is on category, the panel will list down all glossaries along-with the category hierarchy. Here's a list of possible
|
||||
interactions under this view.
|
||||
|
||||
* Expanded view
|
||||
<img src="images/markdown/category_view_2.png" width="250" style="border:2px solid black; margin-left:50px"/>
|
||||
|
||||
* Expanded view
|
||||
<Img src={`/images/markdown/category_view_2.png`} width="300px" />
|
||||
|
||||
|
||||
##### Category context menu
|
||||
|
||||
Clicking on **ellipsis (...)** next to the category name will present a category context menu.
|
||||
|
||||
* To create a new category
|
||||
* To create a new category
|
||||
|
||||
<img src="images/markdown/category_view_glossary_context.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/category_view_glossary_context.png`} width="600px" />
|
||||
|
||||
* To create a sub-category or delete a category
|
||||
* To create a sub-category or delete a category
|
||||
|
||||
<img src="images/markdown/category_view_category_context_1.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/category_view_category_context_1.png`} width="600px" />
|
||||
|
||||
|
||||
##### Category detail page
|
||||
|
||||
Once a category is selected, the details will be presented in the right pane.
|
||||
|
||||
<img src="images/markdown/category_details_with_terms.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/category_details_with_terms.png`} width="600px" />
|
||||
|
||||
|
||||
##### Categorize term
|
||||
|
||||
Click on **+** next to the terms label to link a term under selected category.
|
||||
|
||||
<img src="images/markdown/category_add_term.png" width="600" style="border:2px solid black; margin-left:50px"/>
|
||||
<p/>
|
||||
<img src="images/markdown/category_add_term_1.png" width="500" style="border:2px solid black; margin-left:50px"/>
|
||||
<p/>
|
||||
<img src="images/markdown/category_add_term_2.png" width="500" style="border:2px solid black; margin-left:50px"/>
|
||||
|
||||
<Img src={`/images/markdown/category_add_term.png`} width="600px" />
|
||||
|
||||
<Img src={`/images/markdown/category_add_term_1.png`} width="400px" />
|
||||
|
||||
<Img src={`/images/markdown/category_add_term_2.png`} width="400px" />
|
||||
|
||||
|
||||
#### **Term assignment flow**
|
||||
|
||||
Terms can be assigned to an entity either from the results page or entity details page.
|
||||
|
||||
Terms can be assigned to an entity either from the results page or entity details page.
|
||||
|
||||
##### Assign term
|
||||
|
||||
Click on **+** under "terms" column (if associating term from search page)
|
||||
|
||||
<img src="images/markdown/entity_search_add_term.png" width="" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/entity_search_add_term.png`} />
|
||||
|
||||
|
||||
Click on **+** next to "terms" label (if viewing a specific entity details)
|
||||
|
||||
<img src="images/markdown/entity_details_add_term.png" width="" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/entity_details_add_term.png`} />
|
||||
|
||||
|
||||
Both the actions will present the following modal, follow prompts on screen to complete term assignment.
|
||||
Both the actions will present the following modal, follow prompts on screen to complete term assignment.
|
||||
|
||||
<img src="images/markdown/entity_add_term_modal.png" width="500" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/entity_add_term_modal.png`} width="400px" />
|
||||
|
||||
|
||||
##### Propagated classification
|
||||
|
||||
If a term has classification then the entity is has been assigned inherits the same.
|
||||
|
||||
<img src="images/markdown/term_details_with_classification.png" width="" style="border:2px solid black; margin-left:50px"/>
|
||||
<p/>
|
||||
<img src="images/markdown/entity_assigned_term_with_tag.png" width="500" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/term_details_with_classification.png`} />
|
||||
|
||||
<Img src={`/images/markdown/entity_assigned_term_with_tag.png`} width="600px" />
|
||||
|
||||
|
||||
#### **Search using a term**
|
||||
#### **Search using a term**
|
||||
|
||||
Apache Atlas basic-search API and UI have been updated to support term as a search criteria. This allows users to find
|
||||
entities associated with a given term. Basic search UI now includes a new input drop-down for term based searches.
|
||||
entities associated with a given term. Basic search UI now includes a new input drop-down for term based searches.
|
||||
|
||||
<img src="images/markdown/term_search.png" width="250" style="border:2px solid black; margin-left:50px"/>
|
||||
<Img src={`/images/markdown/term_search.png`} width="300px" />
|
||||
|
||||
|
||||
***
|
||||
|
|
@ -226,8 +221,8 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
##### JSON structure
|
||||
|
||||
* Glossary
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"guid": "2f341934-f18c-48b3-aa12-eaa0a2bfce85",
|
||||
"qualifiedName": "SampleBank",
|
||||
"displayName": "Banking",
|
||||
|
|
@ -268,12 +263,12 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
"relationGuid": "7757b031-4e25-43a8-bf77-946f7f06c67a",
|
||||
"displayText": "Loans"
|
||||
}]
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* Term
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"guid": "e441a540-ee55-4fc8-8eaf-4b9943d8929c",
|
||||
"qualifiedName": "fixed_mtg@SampleBank",
|
||||
"displayName": "15-30 yr mortgage",
|
||||
|
|
@ -304,12 +299,12 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
"termGuid" : "c4e2b956-2589-4648-8596-240d3bea5e44",
|
||||
"displayText": "ARM Loans"
|
||||
}]
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* Category
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"guid": "7f041401-de8c-443f-a3b7-7bf5a910ff6f",
|
||||
"qualifiedName": "Loans.Customer@HortoniaBank",
|
||||
"displayName": "Loans",
|
||||
|
|
@ -332,8 +327,8 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
"relationGuid": "8db1e784-4f04-4eda-9a58-6c9535a95451",
|
||||
"displayText": "ARM loans"
|
||||
}]
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
##### CREATE operations
|
||||
|
||||
|
|
@ -345,7 +340,7 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
6. Create a category with hierarchy
|
||||
7. Create category and categorize term(s)
|
||||
8. Assign term to entities
|
||||
|
||||
|
||||
**NOTE:**
|
||||
|
||||
* During create operations glossary, term and category get an auto assigned GUID and qualifiedName.
|
||||
|
|
@ -361,7 +356,7 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
2. Get all Glossaries - Gives all glossaries with terms and categories (headers) belonging to the respective glossary.
|
||||
3. Get a term by GUID - Gives details about the term, categories it belongs to (if any) and any related term(s).
|
||||
4. Get a category by GUID - Gives details about the category, category hierarchy (if any) and term(s) belonging to the category.
|
||||
5. Get all terms of a given glossary - Gives all terms (with details as mentioned in #3) belonging to given glossary.
|
||||
5. Get all terms of a given glossary - Gives all terms (with details as mentioned in #3) belonging to given glossary.
|
||||
6. Get all categories of a given glossary - Gives all categories (with details as mentioned in #4) belonging to given glossary.
|
||||
7. Get all terms related to given term - Gives all terms related/linked to the given term.
|
||||
8. Get all categories related to a given category (parent and children)
|
||||
|
|
@ -379,13 +374,13 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
6. Update a given category
|
||||
|
||||
**NOTE:**
|
||||
|
||||
* Partial update only deals with the **_primitive attributes_** defined in the Glossary model file.
|
||||
|
||||
* Partial update only deals with the **_primitive attributes_** defined in the Glossary model file.
|
||||
* GUID and qualifiedName can't be changed once assigned. Only way to this is to delete and recreate the required object.
|
||||
* Anchors **can't** be removed in any updates
|
||||
* Update APIs expect the JSON to be modified **in-place** after the **GET** call. Any **_missing_** attributes/relations will be **_deleted_**.
|
||||
* Any update to category's hierarchy leads to a cascaded update of the hierarchy under it e.g. anchor change would affect all children,
|
||||
parent change would affect the qualifiedName of self and children.
|
||||
parent change would affect the qualifiedName of self and children.
|
||||
|
||||
***
|
||||
##### DELETE operations
|
||||
|
|
@ -394,4 +389,3 @@ Following operations are supported by Atlas, the details of REST interface can b
|
|||
2. Delete term - Only deletes the term **_if_** it's not associated/assigned to any entity.
|
||||
3. Delete category - **_Only_** deletes the given category, all children become top-level categories.
|
||||
4. Remove term assignment from entity
|
||||
|
||||
|
|
@ -1,6 +1,17 @@
|
|||
---+ Fault Tolerance and High Availability Options
|
||||
---
|
||||
name: High Availability
|
||||
route: /HighAvailability
|
||||
menu: Documentation
|
||||
submenu: Features
|
||||
---
|
||||
|
||||
---++ Introduction
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Fault Tolerance and High Availability Options
|
||||
|
||||
## Introduction
|
||||
|
||||
Apache Atlas uses and interacts with a variety of systems to provide metadata management and data lineage to data
|
||||
administrators. By choosing and configuring these dependencies appropriately, it is possible to achieve a high degree
|
||||
|
|
@ -8,11 +19,11 @@ of service availability with Atlas. This document describes the state of high av
|
|||
including its capabilities and current limitations, and also the configuration required for achieving this level of
|
||||
high availability.
|
||||
|
||||
[[Architecture][The architecture page]] in the wiki gives an overview of the various components that make up Atlas.
|
||||
[The architecture page](Architecture) in the wiki gives an overview of the various components that make up Atlas.
|
||||
The options mentioned below for various components derive context from the above page, and would be worthwhile to
|
||||
review before proceeding to read this page.
|
||||
|
||||
---++ Atlas Web Service
|
||||
## Atlas Web Service
|
||||
|
||||
Currently, the Atlas Web Service has a limitation that it can only have one active instance at a time. In earlier
|
||||
releases of Atlas, a backup instance could be provisioned and kept available. However, a manual failover was
|
||||
|
|
@ -33,82 +44,87 @@ requests that return information about that instance.
|
|||
|
||||
When configured in a High Availability mode, users can get the following operational benefits:
|
||||
|
||||
* *Uninterrupted service during maintenance intervals*: If an active instance of the Atlas Web Service needs to be brought down for maintenance, another instance would automatically become active and can service requests.
|
||||
* *Uninterrupted service in event of unexpected failures*: If an active instance of the Atlas Web Service fails due to software or hardware errors, another instance would automatically become active and can service requests.
|
||||
* **Uninterrupted service during maintenance intervals**: If an active instance of the Atlas Web Service needs to be brought down for maintenance, another instance would automatically become active and can service requests.
|
||||
* **Uninterrupted service in event of unexpected failures**: If an active instance of the Atlas Web Service fails due to software or hardware errors, another instance would automatically become active and can service requests.
|
||||
|
||||
In the following sub-sections, we describe the steps required to setup High Availability for the Atlas Web Service.
|
||||
We also describe how the deployment and client can be designed to take advantage of this capability.
|
||||
Finally, we describe a few details of the underlying implementation.
|
||||
|
||||
---+++ Setting up the High Availability feature in Atlas
|
||||
### Setting up the High Availability feature in Atlas
|
||||
|
||||
The following pre-requisites must be met for setting up the High Availability feature.
|
||||
|
||||
* Ensure that you install Apache Zookeeper on a cluster of machines (a minimum of 3 servers is recommended for production).
|
||||
* Select 2 or more physical machines to run the Atlas Web Service instances on. These machines define what we refer to as a 'server ensemble' for Atlas.
|
||||
|
||||
To setup High Availability in Atlas, a few configuration options must be defined in the =atlas-application.properties=
|
||||
file. While the complete list of configuration items are defined in the [[Configuration][Configuration Page]], this
|
||||
To setup High Availability in Atlas, a few configuration options must be defined in the `atlas-application.properties`
|
||||
file. While the complete list of configuration items are defined in the [Configuration Page](Configuration), this
|
||||
section lists a few of the main options.
|
||||
|
||||
* High Availability is an optional feature in Atlas. Hence, it must be enabled by setting the configuration option =atlas.server.ha.enabled= to true.
|
||||
* Next, define a list of identifiers, one for each physical machine you have selected for the Atlas Web Service instance. These identifiers can be simple strings like =id1=, =id2= etc. They should be unique and should not contain a comma.
|
||||
* Define a comma separated list of these identifiers as the value of the option =atlas.server.ids=.
|
||||
* For each physical machine, list the IP Address/hostname and port as the value of the configuration =atlas.server.address.id=, where =id= refers to the identifier string for this physical machine.
|
||||
* For e.g., if you have selected 2 machines with hostnames =host1.company.com= and =host2.company.com=, you can define the configuration options as below:
|
||||
<verbatim>
|
||||
atlas.server.ids=id1,id2
|
||||
atlas.server.address.id1=host1.company.com:21000
|
||||
atlas.server.address.id2=host2.company.com:21000
|
||||
</verbatim>
|
||||
* High Availability is an optional feature in Atlas. Hence, it must be enabled by setting the configuration option `atlas.server.ha.enabled` to true.
|
||||
* Next, define a list of identifiers, one for each physical machine you have selected for the Atlas Web Service instance. These identifiers can be simple strings like `id1`, `id2` etc. They should be unique and should not contain a comma.
|
||||
* Define a comma separated list of these identifiers as the value of the option `atlas.server.ids`.
|
||||
* For each physical machine, list the IP Address/hostname and port as the value of the configuration `atlas.server.address.id`, where `id` refers to the identifier string for this physical machine.
|
||||
|
||||
* For e.g., if you have selected 2 machines with hostnames `host1.company.com` and `host2.company.com`, you can define the configuration options as below:
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`atlas.server.ids=id1,id2
|
||||
atlas.server.address.id1=host1.company.com:21000
|
||||
atlas.server.address.id2=host2.company.com:21000`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* Define the Zookeeper quorum which will be used by the Atlas High Availability feature.
|
||||
<verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
atlas.server.ha.zookeeper.connect=zk1.company.com:2181,zk2.company.com:2181,zk3.company.com:2181
|
||||
</verbatim>
|
||||
* You can review other configuration options that are defined for the High Availability feature, and set them up as desired in the =atlas-application.properties= file.
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* You can review other configuration options that are defined for the High Availability feature, and set them up as desired in the `atlas-application.properties` file.
|
||||
* For production environments, the components that Atlas depends on must also be set up in High Availability mode. This is described in detail in the following sections. Follow those instructions to setup and configure them.
|
||||
* Install the Atlas software on the selected physical machines.
|
||||
* Copy the =atlas-application.properties= file created using the steps above to the configuration directory of all the machines.
|
||||
* Copy the `atlas-application.properties` file created using the steps above to the configuration directory of all the machines.
|
||||
* Start the dependent components.
|
||||
* Start each instance of the Atlas Web Service.
|
||||
|
||||
To verify that High Availability is working, run the following script on each of the instances where Atlas Web Service
|
||||
is installed.
|
||||
<verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
$ATLAS_HOME/bin/atlas_admin.py -status
|
||||
</verbatim>
|
||||
</SyntaxHighlighter>
|
||||
This script can print one of the values below as response:
|
||||
|
||||
* *ACTIVE*: This instance is active and can respond to user requests.
|
||||
* *PASSIVE*: This instance is PASSIVE. It will redirect any user requests it receives to the current active instance.
|
||||
* *BECOMING_ACTIVE*: This would be printed if the server is transitioning to become an ACTIVE instance. The server cannot service any metadata user requests in this state.
|
||||
* *BECOMING_PASSIVE*: This would be printed if the server is transitioning to become a PASSIVE instance. The server cannot service any metadata user requests in this state.
|
||||
* **ACTIVE**: This instance is active and can respond to user requests.
|
||||
* **PASSIVE**: This instance is PASSIVE. It will redirect any user requests it receives to the current active instance.
|
||||
* **BECOMING_ACTIVE**: This would be printed if the server is transitioning to become an ACTIVE instance. The server cannot service any metadata user requests in this state.
|
||||
* **BECOMING_PASSIVE**: This would be printed if the server is transitioning to become a PASSIVE instance. The server cannot service any metadata user requests in this state.
|
||||
|
||||
Under normal operating circumstances, only one of these instances should print the value *ACTIVE* as response to
|
||||
the script, and the others would print *PASSIVE*.
|
||||
|
||||
---+++ Configuring clients to use the High Availability feature
|
||||
### Configuring clients to use the High Availability feature
|
||||
|
||||
The Atlas Web Service can be accessed in two ways:
|
||||
|
||||
* *Using the Atlas Web UI*: This is a browser based client that can be used to query the metadata stored in Atlas.
|
||||
* *Using the Atlas REST API*: As Atlas exposes a RESTful API, one can use any standard REST client including libraries in other applications. In fact, Atlas ships with a client called !AtlasClient that can be used as an example to build REST client access.
|
||||
* **Using the Atlas Web UI**: This is a browser based client that can be used to query the metadata stored in Atlas.
|
||||
* **Using the Atlas REST API**: As Atlas exposes a RESTful API, one can use any standard REST client including libraries in other applications. In fact, Atlas ships with a client called !AtlasClient that can be used as an example to build REST client access.
|
||||
|
||||
In order to take advantage of the High Availability feature in the clients, there are two options possible.
|
||||
|
||||
---++++ Using an intermediate proxy
|
||||
#### Using an intermediate proxy
|
||||
|
||||
The simplest solution to enable highly available access to Atlas is to install and configure some intermediate proxy
|
||||
that has a capability to transparently switch services based on status. One such proxy solution is [[http://www.haproxy.org/][HAProxy]].
|
||||
that has a capability to transparently switch services based on status. One such proxy solution is [HAProxy](http://www.haproxy.org/).
|
||||
|
||||
Here is an example HAProxy configuration that can be used. Note this is provided for illustration only, and not as a
|
||||
recommended production configuration. For that, please refer to the HAProxy documentation for appropriate instructions.
|
||||
|
||||
<verbatim>
|
||||
frontend atlas_fe
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`frontend atlas_fe
|
||||
bind *:41000
|
||||
default_backend atlas_be
|
||||
|
||||
backend atlas_be
|
||||
mode http
|
||||
option httpchk get /api/atlas/admin/status
|
||||
|
|
@ -116,23 +132,22 @@ backend atlas_be
|
|||
balance roundrobin
|
||||
server host1_21000 host1:21000 check
|
||||
server host2_21000 host2:21000 check backup
|
||||
|
||||
listen atlas
|
||||
bind localhost:42000
|
||||
</verbatim>
|
||||
bind localhost:42000`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
The above configuration binds HAProxy to listen on port 41000 for incoming client connections. It then routes
|
||||
the connections to either of the hosts host1 or host2 depending on a HTTP status check. The status check is
|
||||
done using a HTTP GET on the REST URL =/api/atlas/admin/status=, and is deemed successful only if the HTTP response
|
||||
done using a HTTP GET on the REST URL `/api/atlas/admin/status`, and is deemed successful only if the HTTP response
|
||||
contains the string ACTIVE.
|
||||
|
||||
---++++ Using automatic detection of active instance
|
||||
#### Using automatic detection of active instance
|
||||
|
||||
If one does not want to setup and manage a separate proxy, then the other option to use the High Availability
|
||||
feature is to build a client application that is capable of detecting status and retrying operations. In such a
|
||||
setting, the client application can be launched with the URLs of all Atlas Web Service instances that form the
|
||||
ensemble. The client should then call the REST URL =/api/atlas/admin/status= on each of these to determine which is
|
||||
the active instance. The response from the Active instance would be of the form ={Status:ACTIVE}=. Also, when the
|
||||
ensemble. The client should then call the REST URL `/api/atlas/admin/status` on each of these to determine which is
|
||||
the active instance. The response from the Active instance would be of the form `{Status:ACTIVE}`. Also, when the
|
||||
client faces any exceptions in the course of an operation, it should again determine which of the remaining URLs
|
||||
is active and retry the operation.
|
||||
|
||||
|
|
@ -143,56 +158,57 @@ Utilities in Atlas, like =quick_start.py= and =import-hive.sh= can be configured
|
|||
URLs. When launched in this mode, the !AtlasClient automatically selects and works with the current active instance.
|
||||
If a proxy is set up in between, then its address can be used when running quick_start.py or import-hive.sh.
|
||||
|
||||
---+++ Implementation Details of Atlas High Availability
|
||||
### Implementation Details of Atlas High Availability
|
||||
|
||||
The Atlas High Availability work is tracked under the master JIRA [[https://issues.apache.org/jira/browse/ATLAS-510][ATLAS-510]].
|
||||
The Atlas High Availability work is tracked under the master JIRA
|
||||
[ATLAS-510](https://issues.apache.org/jira/browse/ATLAS-510).
|
||||
The JIRAs filed under it have detailed information about how the High Availability feature has been implemented.
|
||||
At a high level the following points can be called out:
|
||||
|
||||
* The automatic selection of an Active instance, as well as automatic failover to a new Active instance happen through a leader election algorithm.
|
||||
* For leader election, we use the [[http://curator.apache.org/curator-recipes/leader-latch.html][Leader Latch Recipe]] of [[http://curator.apache.org][Apache Curator]].
|
||||
* For leader election, we use the [Leader Latch Recipe](http://curator.apache.org/curator-recipes/leader-latch.html) of [Apache Curator](http://curator.apache.org)
|
||||
* The Active instance is the only one which initializes, modifies or reads state in the backend stores to keep them consistent.
|
||||
* Also, when an instance is elected as Active, it refreshes any cached information from the backend stores to get up to date.
|
||||
* A servlet filter ensures that only the active instance services user requests. If a passive instance receives these requests, it automatically redirects them to the current active instance.
|
||||
|
||||
---++ Metadata Store
|
||||
## Metadata Store
|
||||
|
||||
As described above, Atlas uses JanusGraph to store the metadata it manages. By default, Atlas uses a standalone HBase
|
||||
instance as the backing store for JanusGraph. In order to provide HA for the metadata store, we recommend that Atlas be
|
||||
configured to use distributed HBase as the backing store for JanusGraph. Doing this implies that you could benefit from the
|
||||
HA guarantees HBase provides. In order to configure Atlas to use HBase in HA mode, do the following:
|
||||
|
||||
* Choose an existing HBase cluster that is set up in HA mode to configure in Atlas (OR) Set up a new HBase cluster in [[http://hbase.apache.org/book.html#quickstart_fully_distributed][HA mode]].
|
||||
* If setting up HBase for Atlas, please following instructions listed for setting up HBase in the [[InstallationSteps][Installation Steps]].
|
||||
* Choose an existing HBase cluster that is set up in HA mode to configure in Atlas (OR) Set up a new HBase cluster in [HA mode](http://hbase.apache.org/book.html#quickstart_fully_distributed).
|
||||
* If setting up HBase for Atlas, please following instructions listed for setting up HBase in the [Installation Steps](InstallationSteps).
|
||||
* We recommend using more than one HBase masters (at least 2) in the cluster on different physical hosts that use Zookeeper for coordination to provide redundancy and high availability of HBase.
|
||||
* Refer to the [[Configuration][Configuration page]] for the options to configure in atlas.properties to setup Atlas with HBase.
|
||||
* Refer to the [Configuration page](Configuration) for the options to configure in atlas.properties to setup Atlas with HBase.
|
||||
|
||||
---++ Index Store
|
||||
## Index Store
|
||||
|
||||
As described above, Atlas indexes metadata through JanusGraph to support full text search queries. In order to provide HA
|
||||
for the index store, we recommend that Atlas be configured to use Solr or Elasticsearch as the backing index store for JanusGraph.
|
||||
|
||||
---+++ Solr
|
||||
### Solr
|
||||
In order to configure Atlas to use Solr in HA mode, do the following:
|
||||
|
||||
* Choose an existing !SolrCloud cluster setup in HA mode to configure in Atlas (OR) Set up a new [[https://cwiki.apache.org/confluence/display/solr/SolrCloud][SolrCloud cluster]].
|
||||
* Choose an existing !SolrCloud cluster setup in HA mode to configure in Atlas (OR) Set up a new [SolrCloud cluster](https://cwiki.apache.org/confluence/display/solr/SolrCloud).
|
||||
* Ensure Solr is brought up on at least 2 physical hosts for redundancy, and each host runs a Solr node.
|
||||
* We recommend the number of replicas to be set to at least 2 for redundancy.
|
||||
* Create the !SolrCloud collections required by Atlas, as described in [[InstallationSteps][Installation Steps]]
|
||||
* Refer to the [[Configuration][Configuration page]] for the options to configure in atlas.properties to setup Atlas with Solr.
|
||||
* Create the !SolrCloud collections required by Atlas, as described in [Installation Steps](InstallationSteps)
|
||||
* Refer to the [Configuration page](Configuration) for the options to configure in atlas.properties to setup Atlas with Solr.
|
||||
|
||||
---+++ Elasticsearch (Tech Preview)
|
||||
### Elasticsearch (Tech Preview)
|
||||
In order to configure Atlas to use Elasticsearch in HA mode, do the following:
|
||||
|
||||
* Choose an existing Elasticsearch cluster setup, (OR) setup a new cluster [[https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup.html][Elasticsearch cluster]].
|
||||
* Choose an existing Elasticsearch cluster setup, (OR) setup a new cluster [Elasticsearch cluster](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup.html).
|
||||
* Ensure that Elasticsearch is brought up on at least five physical hosts for redundancy.
|
||||
* A replica count of 3 is recommended
|
||||
* Refer to the [[Configuration][Configuration page]] for the options to configure in atlas.properties to setup Atlas with Elasticsearch.
|
||||
* Refer to the [Configuration page](Configuration) for the options to configure in atlas.properties to setup Atlas with Elasticsearch.
|
||||
|
||||
---++ Notification Server
|
||||
## Notification Server
|
||||
|
||||
Metadata notification events from Hooks are sent to Atlas by writing them to a Kafka topic called *ATLAS_HOOK*. Similarly, events from
|
||||
Atlas to other integrating components like Ranger, are written to a Kafka topic called *ATLAS_ENTITIES*. Since Kafka
|
||||
Metadata notification events from Hooks are sent to Atlas by writing them to a Kafka topic called **ATLAS_HOOK**. Similarly, events from
|
||||
Atlas to other integrating components like Ranger, are written to a Kafka topic called **ATLAS_ENTITIES**. Since Kafka
|
||||
persists these messages, the events will not be lost even if the consumers are down as the events are being sent. In
|
||||
addition, we recommend Kafka is also setup for fault tolerance so that it has higher availability guarantees. In order
|
||||
to configure Atlas to use Kafka in HA mode, do the following:
|
||||
|
|
@ -204,18 +220,21 @@ to configure Atlas to use Kafka in HA mode, do the following:
|
|||
* The number of partitions for the ATLAS topics should be set to 1 (numPartitions)
|
||||
* Decide number of replicas for Kafka topic: Set this to at least 2 for redundancy.
|
||||
* Run the following commands:
|
||||
<verbatim>
|
||||
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper <list of zookeeper host:port entries> --topic ATLAS_HOOK --replication-factor <numReplicas> --partitions 1
|
||||
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper <list of zookeeper host:port entries> --topic ATLAS_ENTITIES --replication-factor <numReplicas> --partitions 1
|
||||
Here KAFKA_HOME points to the Kafka installation directory.
|
||||
</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper <list of zookeeper host:port entries> --topic ATLAS_HOOK --replication-factor <numReplicas> --partitions 1
|
||||
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper <list of zookeeper host:port entries> --topic ATLAS_ENTITIES --replication-factor <numReplicas> --partitions 1
|
||||
Here KAFKA_HOME points to the Kafka installation directory.`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* In atlas-application.properties, set the following configuration:
|
||||
<verbatim>
|
||||
atlas.notification.embedded=false
|
||||
atlas.kafka.zookeeper.connect=<comma separated list of servers forming Zookeeper quorum used by Kafka>
|
||||
atlas.kafka.bootstrap.servers=<comma separated list of Kafka broker endpoints in host:port form> - Give at least 2 for redundancy.
|
||||
</verbatim>
|
||||
|
||||
---++ Known Issues
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`atlas.notification.embedded=false
|
||||
atlas.kafka.zookeeper.connect=<comma separated list of servers forming Zookeeper quorum used by Kafka>
|
||||
atlas.kafka.bootstrap.servers=<comma separated list of Kafka broker endpoints in host:port form> - Give at least 2 for redundancy.`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* If the HBase region servers hosting the Atlas table are down, Atlas would not be able to store or retrieve metadata from HBase until they are brought back online.
|
||||
## Known Issues
|
||||
|
||||
* If the HBase region servers hosting the Atlas table are down, Atlas would not be able to store or retrieve metadata from HBase until they are brought back online.
|
||||
55
docs/src/site/twiki/Architecture.twiki → docs/src/documents/HighLevelArchitecture.md
Executable file → Normal file
55
docs/src/site/twiki/Architecture.twiki → docs/src/documents/HighLevelArchitecture.md
Executable file → Normal file
|
|
@ -1,16 +1,24 @@
|
|||
---+ Architecture
|
||||
---
|
||||
name: High Level Architecture
|
||||
route: /Architecture
|
||||
menu: Documentation
|
||||
submenu: Features
|
||||
---
|
||||
import Img from 'theme/components/shared/Img'
|
||||
|
||||
---++ Introduction
|
||||
# Architecture
|
||||
|
||||
---++ Atlas High Level Architecture - Overview
|
||||
<img src="images/twiki/architecture.png" height="400" width="600" />
|
||||
## Introduction
|
||||
|
||||
## Atlas High Level Architecture - Overview
|
||||
<Img src={`/images/twiki/architecture.png`} width="800" />
|
||||
|
||||
The components of Atlas can be grouped under the following major categories:
|
||||
|
||||
---+++ Core
|
||||
### Core
|
||||
Atlas core includes the following components:
|
||||
|
||||
*Type System*: Atlas allows users to define a model for the metadata objects they want to manage. The model is composed
|
||||
**Type System**: Atlas allows users to define a model for the metadata objects they want to manage. The model is composed
|
||||
of definitions called ‘types’. Instances of ‘types’ called ‘entities’ represent the actual metadata objects that are
|
||||
managed. The Type System is a component that allows users to define and manage the types and entities. All metadata
|
||||
objects managed by Atlas out of the box (like Hive tables, for e.g.) are modelled using types and represented as
|
||||
|
|
@ -20,60 +28,53 @@ One key point to note is that the generic nature of the modelling in Atlas allow
|
|||
define both technical metadata and business metadata. It is also possible to define rich relationships between the
|
||||
two using features of Atlas.
|
||||
|
||||
*Graph Engine*: Internally, Atlas persists metadata objects it manages using a Graph model. This approach provides great
|
||||
**Graph Engine**: Internally, Atlas persists metadata objects it manages using a Graph model. This approach provides great
|
||||
flexibility and enables efficient handling of rich relationships between the metadata objects. Graph engine component is
|
||||
responsible for translating between types and entities of the Atlas type system, and the underlying graph persistence model.
|
||||
In addition to managing the graph objects, the graph engine also creates the appropriate indices for the metadata
|
||||
objects so that they can be searched efficiently. Atlas uses the JanusGraph to store the metadata objects.
|
||||
|
||||
*Ingest / Export*: The Ingest component allows metadata to be added to Atlas. Similarly, the Export component exposes
|
||||
**Ingest / Export**: The Ingest component allows metadata to be added to Atlas. Similarly, the Export component exposes
|
||||
metadata changes detected by Atlas to be raised as events. Consumers can consume these change events to react to
|
||||
metadata changes in real time.
|
||||
|
||||
|
||||
---+++ Integration
|
||||
### Integration
|
||||
Users can manage metadata in Atlas using two methods:
|
||||
|
||||
*API*: All functionality of Atlas is exposed to end users via a REST API that allows types and entities to be created,
|
||||
**API**: All functionality of Atlas is exposed to end users via a REST API that allows types and entities to be created,
|
||||
updated and deleted. It is also the primary mechanism to query and discover the types and entities managed by Atlas.
|
||||
|
||||
*Messaging*: In addition to the API, users can choose to integrate with Atlas using a messaging interface that is
|
||||
**Messaging**: In addition to the API, users can choose to integrate with Atlas using a messaging interface that is
|
||||
based on Kafka. This is useful both for communicating metadata objects to Atlas, and also to consume metadata change
|
||||
events from Atlas using which applications can be built. The messaging interface is particularly useful if one wishes
|
||||
to use a more loosely coupled integration with Atlas that could allow for better scalability, reliability etc. Atlas
|
||||
uses Apache Kafka as a notification server for communication between hooks and downstream consumers of metadata
|
||||
notification events. Events are written by the hooks and Atlas to different Kafka topics.
|
||||
|
||||
---+++ Metadata sources
|
||||
### Metadata sources
|
||||
Atlas supports integration with many sources of metadata out of the box. More integrations will be added in future
|
||||
as well. Currently, Atlas supports ingesting and managing metadata from the following sources:
|
||||
|
||||
* [[Hook-HBase][HBase]]
|
||||
* [[Hook-Hive][Hive]]
|
||||
* [[Hook-Sqoop][Sqoop]]
|
||||
* [[Hook-Storm][Storm]]
|
||||
* [[Bridge-Kafka][Kafka]]
|
||||
* [HBase](/Hook-HBase)
|
||||
* [Hive](/Hook-Hive)
|
||||
* [Sqoop](/Hook-Sqoop)
|
||||
* [Storm](/Hook-Storm)
|
||||
* [Kafka](/Hook-Kafka)
|
||||
|
||||
The integration implies two things:
|
||||
There are metadata models that Atlas defines natively to represent objects of these components.
|
||||
There are components Atlas provides to ingest metadata objects from these components
|
||||
(in real time or in batch mode in some cases).
|
||||
|
||||
---+++ Applications
|
||||
### Applications
|
||||
The metadata managed by Atlas is consumed by a variety of applications for satisfying many governance use cases.
|
||||
|
||||
*Atlas Admin UI*: This component is a web based application that allows data stewards and scientists to discover
|
||||
**Atlas Admin UI**: This component is a web based application that allows data stewards and scientists to discover
|
||||
and annotate metadata. Of primary importance here is a search interface and SQL like query language that can be
|
||||
used to query the metadata types and objects managed by Atlas. The Admin UI uses the REST API of Atlas for
|
||||
building its functionality.
|
||||
|
||||
*Tag Based Policies*: [[http://ranger.apache.org/][Apache Ranger]] is an advanced security management solution
|
||||
**Tag Based Policies**: [Apache Ranger](http://ranger.apache.org/) is an advanced security management solution
|
||||
for the Hadoop ecosystem having wide integration with a variety of Hadoop components. By integrating with Atlas,
|
||||
Ranger allows security administrators to define metadata driven security policies for effective governance.
|
||||
Ranger is a consumer to the metadata change events notified by Atlas.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,6 +1,17 @@
|
|||
---+ Falcon Atlas Bridge
|
||||
---
|
||||
name: Falcon
|
||||
route: /Hook-Falcon
|
||||
menu: Documentation
|
||||
submenu: Hooks
|
||||
---
|
||||
|
||||
---++ Falcon Model
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Falcon Atlas Bridge
|
||||
|
||||
## Falcon Model
|
||||
The default hive model includes the following types:
|
||||
* Entity types:
|
||||
* falcon_cluster
|
||||
|
|
@ -22,25 +33,28 @@ The default hive model includes the following types:
|
|||
One falcon_process entity is created for every cluster that the falcon process is defined for.
|
||||
|
||||
The entities are created and de-duped using unique qualifiedName attribute. They provide namespace and can be used for querying/lineage as well. The unique attributes are:
|
||||
* falcon_process.qualifiedName - <process name>@<cluster name>
|
||||
* falcon_cluster.qualifiedName - <cluster name>
|
||||
* falcon_feed.qualifiedName - <feed name>@<cluster name>
|
||||
* falcon_feed_creation.qualifiedName - <feed name>
|
||||
* falcon_feed_replication.qualifiedName - <feed name>
|
||||
* falcon_process.qualifiedName - `<process name>@<cluster name>`
|
||||
* falcon_cluster.qualifiedName - `<cluster name>`
|
||||
* falcon_feed.qualifiedName - `<feed name>@<cluster name>`
|
||||
* falcon_feed_creation.qualifiedName - `<feed name>`
|
||||
* falcon_feed_replication.qualifiedName - `<feed name>`
|
||||
|
||||
---++ Falcon Hook
|
||||
## Falcon Hook
|
||||
Falcon supports listeners on falcon entity submission. This is used to add entities in Atlas using the model detailed above.
|
||||
Follow the instructions below to setup Atlas hook in Falcon:
|
||||
* Add 'org.apache.atlas.falcon.service.AtlasService' to application.services in <falcon-conf>/startup.properties
|
||||
* Add 'org.apache.atlas.falcon.service.AtlasService' to application.services in `<falcon-conf>`/startup.properties
|
||||
* untar apache-atlas-${project.version}-falcon-hook.tar.gz
|
||||
* cd apache-atlas-falcon-hook-${project.version}
|
||||
* Copy entire contents of folder apache-atlas-falcon-hook-${project.version}/hook/falcon to <atlas-home>/hook/falcon
|
||||
* Link Atlas hook jars in Falcon classpath - 'ln -s <atlas-home>/hook/falcon/* <falcon-home>/server/webapp/falcon/WEB-INF/lib/'
|
||||
* In <falcon_conf>/falcon-env.sh, set an environment variable as follows:
|
||||
<verbatim>
|
||||
export FALCON_SERVER_OPTS="<atlas_home>/hook/falcon/*:$FALCON_SERVER_OPTS"</verbatim>
|
||||
* Copy entire contents of folder apache-atlas-falcon-hook-${project.version}/hook/falcon to `<atlas-home>`/hook/falcon
|
||||
* Link Atlas hook jars in Falcon classpath - 'ln -s `<atlas-home>`/hook/falcon/* `<falcon-home>`/server/webapp/falcon/WEB-INF/lib/'
|
||||
* In `<falcon_conf>`/falcon-env.sh, set an environment variable as follows:
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`export FALCON_SERVER_OPTS="<atlas_home>/hook/falcon/*:$FALCON_SERVER_OPTS"`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
The following properties in <atlas-conf>/atlas-application.properties control the thread pool and notification details:
|
||||
|
||||
The following properties in `<atlas-conf>`/atlas-application.properties control the thread pool and notification details:
|
||||
* atlas.hook.falcon.synchronous - boolean, true to run the hook synchronously. default false
|
||||
* atlas.hook.falcon.numRetries - number of retries for notification failure. default 3
|
||||
* atlas.hook.falcon.minThreads - core number of threads. default 5
|
||||
|
|
@ -48,8 +62,8 @@ The following properties in <atlas-conf>/atlas-application.properties control th
|
|||
* atlas.hook.falcon.keepAliveTime - keep alive time in msecs. default 10
|
||||
* atlas.hook.falcon.queueSize - queue size for the threadpool. default 10000
|
||||
|
||||
Refer [[Configuration][Configuration]] for notification related configurations
|
||||
Refer [Configuration](Configuration) for notification related configurations
|
||||
|
||||
|
||||
---++ NOTES
|
||||
## NOTES
|
||||
* In falcon cluster entity, cluster name used should be uniform across components like hive, falcon, sqoop etc. If used with ambari, ambari cluster name should be used for cluster entity
|
||||
|
|
@ -1,6 +1,17 @@
|
|||
---+ Apache Atlas Hook & Bridge for Apache HBase
|
||||
---
|
||||
name: HBase
|
||||
route: /Hook-HBase
|
||||
menu: Documentation
|
||||
submenu: Hooks
|
||||
---
|
||||
|
||||
---++ HBase Model
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Apache Atlas Hook & Bridge for Apache HBase
|
||||
|
||||
## HBase Model
|
||||
HBase model includes the following types:
|
||||
* Entity types:
|
||||
* hbase_namespace
|
||||
|
|
@ -14,60 +25,63 @@ HBase model includes the following types:
|
|||
* attributes: qualifiedName, name, description, owner, columns, createTime, bloomFilterType, compressionType, compactionCompressionType, encryptionType, inMemoryCompactionPolicy, keepDeletedCells, maxversions, minVersions, datablockEncoding, storagePolicy, ttl, blockCachedEnabled, cacheBloomsOnWrite, cacheDataOnWrite, evictBlocksOnClose, prefetchBlocksOnOpen, newVersionsBehavior, isMobEnabled, mobCompactPartitionPolicy
|
||||
|
||||
HBase entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Note that namespaceName, tableName and columnFamilyName should be in lower case.
|
||||
<verbatim>
|
||||
hbase_namespace.qualifiedName: <namespaceName>@<clusterName>
|
||||
hbase_table.qualifiedName: <namespaceName>:<tableName>@<clusterName>
|
||||
hbase_column_family.qualifiedName: <namespaceName>:<tableName>.<columnFamilyName>@<clusterName>
|
||||
</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`hbase_namespace.qualifiedName: <namespaceName>@<clusterName>
|
||||
hbase_table.qualifiedName: <namespaceName>:<tableName>@<clusterName>
|
||||
hbase_column_family.qualifiedName: <namespaceName>:<tableName>.<columnFamilyName>@<clusterName>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
---++ HBase Hook
|
||||
## HBase Hook
|
||||
Atlas HBase hook registers with HBase master as a co-processor. On detecting changes to HBase namespaces/tables/column-families, Atlas hook updates the metadata in Atlas via Kafka notifications.
|
||||
Follow the instructions below to setup Atlas hook in HBase:
|
||||
* Register Atlas hook in hbase-site.xml by adding the following:
|
||||
<verbatim>
|
||||
<property>
|
||||
<name>hbase.coprocessor.master.classes</name>
|
||||
<value>org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor</value>
|
||||
</property></verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="xml" style={theme.dark}>
|
||||
{`<property>
|
||||
<name>hbase.coprocessor.master.classes</name>
|
||||
<value>org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor</value>
|
||||
</property>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* untar apache-atlas-${project.version}-hbase-hook.tar.gz
|
||||
* cd apache-atlas-hbase-hook-${project.version}
|
||||
* Copy entire contents of folder apache-atlas-hbase-hook-${project.version}/hook/hbase to <atlas package>/hook/hbase
|
||||
* Link Atlas hook jars in HBase classpath - 'ln -s <atlas package>/hook/hbase/* <hbase-home>/lib/'
|
||||
* Copy <atlas-conf>/atlas-application.properties to the HBase conf directory.
|
||||
* Copy entire contents of folder apache-atlas-hbase-hook-${project.version}/hook/hbase to `<atlas package>`/hook/hbase
|
||||
* Link Atlas hook jars in HBase classpath - 'ln -s `<atlas package>`/hook/hbase/* `<hbase-home>`/lib/'
|
||||
* Copy `<atlas-conf>`/atlas-application.properties to the HBase conf directory.
|
||||
|
||||
The following properties in atlas-application.properties control the thread pool and notification details:
|
||||
<verbatim>
|
||||
atlas.hook.hbase.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in HBase operations. Default: false
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`atlas.hook.hbase.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in HBase operations. Default: false
|
||||
atlas.hook.hbase.numRetries=3 # number of retries for notification failure. Default: 3
|
||||
atlas.hook.hbase.queueSize=10000 # queue size for the threadpool. Default: 10000
|
||||
|
||||
atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary
|
||||
|
||||
atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181
|
||||
atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000
|
||||
atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000
|
||||
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20
|
||||
</verbatim>
|
||||
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.".
|
||||
For list of configuration supported by Kafka producer, please refer to [[http://kafka.apache.org/documentation/#producerconfigs][Kafka Producer Configs]]
|
||||
For list of configuration supported by Kafka producer, please refer to [Kafka Producer Configs](http://kafka.apache.org/documentation/#producerconfigs)
|
||||
|
||||
---++ NOTES
|
||||
## NOTES
|
||||
* Only the namespace, table and column-family create/update/ delete operations are captured by Atlas HBase hook. Changes to columns are be captured.
|
||||
|
||||
|
||||
---++ Importing HBase Metadata
|
||||
## Importing HBase Metadata
|
||||
Apache Atlas provides a command-line utility, import-hbase.sh, to import metadata of Apache HBase namespaces and tables into Apache Atlas.
|
||||
This utility can be used to initialize Apache Atlas with namespaces/tables present in a Apache HBase cluster.
|
||||
This utility supports importing metadata of a specific table, tables in a specific namespace or all tables.
|
||||
|
||||
<verbatim>
|
||||
Usage 1: <atlas package>/hook-bin/import-hbase.sh
|
||||
<SyntaxHighlighter wrapLines={true} language="java" style={theme.dark}>
|
||||
{`Usage 1: <atlas package>/hook-bin/import-hbase.sh
|
||||
Usage 2: <atlas package>/hook-bin/import-hbase.sh [-n <namespace regex> OR --namespace <namespace regex>] [-t <table regex> OR --table <table regex>]
|
||||
Usage 3: <atlas package>/hook-bin/import-hbase.sh [-f <filename>]
|
||||
File Format:
|
||||
namespace1:tbl1
|
||||
namespace1:tbl2
|
||||
namespace2:tbl1
|
||||
</verbatim>
|
||||
namespace2:tbl1`}
|
||||
</SyntaxHighlighter>
|
||||
|
|
@ -1,12 +1,25 @@
|
|||
---+ Apache Atlas Hook & Bridge for Apache Hive
|
||||
---
|
||||
name: Hive
|
||||
route: /Hook-Hive
|
||||
menu: Documentation
|
||||
submenu: Hooks
|
||||
---
|
||||
|
||||
---++ Hive Model
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
import Img from 'theme/components/shared/Img'
|
||||
|
||||
# Apache Atlas Hook & Bridge for Apache Hive
|
||||
|
||||
|
||||
## Hive Model
|
||||
Hive model includes the following types:
|
||||
* Entity types:
|
||||
* hive_db
|
||||
* super-types: !Asset
|
||||
* attributes: qualifiedName, name, description, owner, clusterName, location, parameters, ownerName
|
||||
* hive_table
|
||||
* hive_db
|
||||
* super-types: !Asset
|
||||
* attributes: qualifiedName, name, description, owner, clusterName, location, parameters, ownerName
|
||||
* hive_table
|
||||
* super-types: !DataSet
|
||||
* attributes: qualifiedName, name, description, owner, db, createTime, lastAccessTime, comment, retention, sd, partitionKeys, columns, aliases, parameters, viewOriginalText, viewExpandedText, tableType, temporary
|
||||
* hive_column
|
||||
|
|
@ -21,63 +34,69 @@ Hive model includes the following types:
|
|||
* hive_column_lineage
|
||||
* super-types: Process
|
||||
* attributes: qualifiedName, name, description, owner, inputs, outputs, query, depenendencyType, expression
|
||||
|
||||
|
||||
* Enum types:
|
||||
* hive_principal_type
|
||||
* values: USER, ROLE, GROUP
|
||||
* hive_principal_type
|
||||
* values: USER, ROLE, GROUP
|
||||
|
||||
|
||||
* Struct types:
|
||||
* hive_order
|
||||
* hive_order
|
||||
* attributes: col, order
|
||||
* hive_serde
|
||||
* attributes: name, serializationLib, parameters
|
||||
|
||||
|
||||
Hive entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below. Note that dbName, tableName and columnName should be in lower case.
|
||||
<verbatim>
|
||||
hive_db.qualifiedName: <dbName>@<clusterName>
|
||||
hive_table.qualifiedName: <dbName>.<tableName>@<clusterName>
|
||||
hive_column.qualifiedName: <dbName>.<tableName>.<columnName>@<clusterName>
|
||||
hive_process.queryString: trimmed query string in lower case
|
||||
</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`hive_db.qualifiedName: <dbName>@<clusterName>
|
||||
hive_table.qualifiedName: <dbName>.<tableName>@<clusterName>
|
||||
hive_column.qualifiedName: <dbName>.<tableName>.<columnName>@<clusterName>
|
||||
hive_process.queryString: trimmed query string in lower case`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
---++ Hive Hook
|
||||
## Hive Hook
|
||||
Atlas Hive hook registers with Hive to listen for create/update/delete operations and updates the metadata in Atlas, via Kafka notifications, for the changes in Hive.
|
||||
Follow the instructions below to setup Atlas hook in Hive:
|
||||
* Set-up Atlas hook in hive-site.xml by adding the following:
|
||||
<verbatim>
|
||||
<property>
|
||||
<name>hive.exec.post.hooks</name>
|
||||
* Set-up Atlas hook in hive-site.xml by adding the following:
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="xml" style={theme.dark}>
|
||||
{`<property>
|
||||
<name>hive.exec.post.hooks</name>
|
||||
<value>org.apache.atlas.hive.hook.HiveHook</value>
|
||||
</property></verbatim>
|
||||
</property>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* untar apache-atlas-${project.version}-hive-hook.tar.gz
|
||||
* cd apache-atlas-hive-hook-${project.version}
|
||||
* Copy entire contents of folder apache-atlas-hive-hook-${project.version}/hook/hive to <atlas package>/hook/hive
|
||||
* Add 'export HIVE_AUX_JARS_PATH=<atlas package>/hook/hive' in hive-env.sh of your hive configuration
|
||||
* Copy <atlas-conf>/atlas-application.properties to the hive conf directory.
|
||||
* untar apache-atlas-${project.version}-hive-hook.tar.gz
|
||||
* cd apache-atlas-hive-hook-${project.version}
|
||||
* Copy entire contents of folder apache-atlas-hive-hook-${project.version}/hook/hive to `<atlas package>`/hook/hive
|
||||
* Add 'export HIVE_AUX_JARS_PATH=`<atlas package>`/hook/hive' in hive-env.sh of your hive configuration
|
||||
* Copy `<atlas-conf>`/atlas-application.properties to the hive conf directory.
|
||||
|
||||
|
||||
The following properties in atlas-application.properties control the thread pool and notification details:
|
||||
<verbatim>
|
||||
atlas.hook.hive.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Hive query completion. Default: false
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`atlas.hook.hive.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Hive query completion. Default: false
|
||||
atlas.hook.hive.numRetries=3 # number of retries for notification failure. Default: 3
|
||||
atlas.hook.hive.queueSize=10000 # queue size for the threadpool. Default: 10000
|
||||
|
||||
atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary
|
||||
|
||||
atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181
|
||||
atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000
|
||||
atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000
|
||||
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20
|
||||
</verbatim>
|
||||
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [[http://kafka.apache.org/documentation/#producerconfigs][Kafka Producer Configs]]
|
||||
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [Kafka Producer Configs](http://kafka.apache.org/documentation/#producerconfigs)
|
||||
|
||||
---++ Column Level Lineage
|
||||
## Column Level Lineage
|
||||
|
||||
Starting from 0.8-incubating version of Atlas, Column level lineage is captured in Atlas. Below are the details
|
||||
|
||||
---+++ Model
|
||||
### Model
|
||||
* !ColumnLineageProcess type is a subtype of Process
|
||||
|
||||
* This relates an output Column to a set of input Columns or the Input Table
|
||||
|
|
@ -91,23 +110,24 @@ Starting from 0.8-incubating version of Atlas, Column level lineage is captured
|
|||
|
||||
* Since Process links input and output !DataSets, Column is a subtype of !DataSet
|
||||
|
||||
---+++ Examples
|
||||
### Examples
|
||||
For a simple CTAS below:
|
||||
<verbatim>
|
||||
create table t2 as select id, name from T1</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="sql" style={theme.dark}>
|
||||
create table t2 as select id, name from T1
|
||||
</SyntaxHighlighter>
|
||||
|
||||
The lineage is captured as
|
||||
|
||||
<img src="images/column_lineage_ex1.png" height="200" width="400" />
|
||||
<Img src={`/images/column_lineage_ex1.png`} height="200" width="400" />
|
||||
|
||||
|
||||
|
||||
---+++ Extracting Lineage from Hive commands
|
||||
### Extracting Lineage from Hive commands
|
||||
* The !HiveHook maps the !LineageInfo in the !HookContext to Column lineage instances
|
||||
|
||||
* The !LineageInfo in Hive provides column-level lineage for the final !FileSinkOperator, linking them to the input columns in the Hive Query
|
||||
|
||||
---++ NOTES
|
||||
## NOTES
|
||||
* Column level lineage works with Hive version 1.2.1 after the patch for <a href="https://issues.apache.org/jira/browse/HIVE-13112">HIVE-13112</a> is applied to Hive source
|
||||
* Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names
|
||||
* The following hive operations are captured by hive hook currently
|
||||
|
|
@ -120,17 +140,17 @@ The lineage is captured as
|
|||
* alter view
|
||||
|
||||
|
||||
---++ Importing Hive Metadata
|
||||
## Importing Hive Metadata
|
||||
Apache Atlas provides a command-line utility, import-hive.sh, to import metadata of Apache Hive databases and tables into Apache Atlas.
|
||||
This utility can be used to initialize Apache Atlas with databases/tables present in Apache Hive.
|
||||
This utility supports importing metadata of a specific table, tables in a specific database or all databases and tables.
|
||||
|
||||
<verbatim>
|
||||
Usage 1: <atlas package>/hook-bin/import-hive.sh
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`Usage 1: <atlas package>/hook-bin/import-hive.sh
|
||||
Usage 2: <atlas package>/hook-bin/import-hive.sh [-d <database regex> OR --database <database regex>] [-t <table regex> OR --table <table regex>]
|
||||
Usage 3: <atlas package>/hook-bin/import-hive.sh [-f <filename>]
|
||||
File Format:
|
||||
database1:tbl1
|
||||
database1:tbl2
|
||||
database2:tbl1
|
||||
</verbatim>
|
||||
database2:tbl1`}
|
||||
</SyntaxHighlighter>
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
---
|
||||
name: Kafka
|
||||
route: /Hook-Kafka
|
||||
menu: Documentation
|
||||
submenu: Hooks
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
import Img from 'theme/components/shared/Img'
|
||||
|
||||
|
||||
# Apache Atlas Hook for Apache Kafka
|
||||
|
||||
## Kafka Model
|
||||
Kafka model includes the following types:
|
||||
* Entity types:
|
||||
* kafka_topic
|
||||
* super-types: !DataSet
|
||||
* attributes: qualifiedName, name, description, owner, topic, uri, partitionCount
|
||||
|
||||
Kafka entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below.
|
||||
Note that qualifiedName will have topic name in lower case.
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`topic.qualifiedName: <topic>@<clusterName>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
## Setup
|
||||
Binary files are present in apache-atlas-<release-version/>-kafka-hook.tar.gz
|
||||
|
||||
Copy apache-atlas-kafka-hook-<release-version/>/hook/kafka folder to <atlas package/>/hook/ directory
|
||||
|
||||
Copy apache-atlas-kafka-hook-<release-version/>/hook-bin folder to <atlas package/>/hook-bin directory
|
||||
|
||||
## Importing Kafka Metadata
|
||||
Apache Atlas provides a command-line utility, import-kafka.sh, to import metadata of Apache Kafka topics into Apache Atlas.
|
||||
This utility can be used to initialize Apache Atlas with topics present in Apache Kafka.
|
||||
This utility supports importing metadata of a specific topic or all topics.
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`sage 1: <atlas package>/hook-bin/import-kafka.sh
|
||||
Usage 2: <atlas package>/hook-bin/import-kafka.sh [-t <topic prefix> OR --topic <topic prefix>]
|
||||
Usage 3: <atlas package>/hook-bin/import-kafka.sh [-f <filename>]
|
||||
File Format:
|
||||
topic1
|
||||
topic2
|
||||
topic3`}
|
||||
</SyntaxHighlighter>
|
||||
|
|
@ -1,6 +1,17 @@
|
|||
---+ Apache Atlas Hook for Apache Sqoop
|
||||
---
|
||||
name: Sqoop
|
||||
route: /Hook-Sqoop
|
||||
menu: Documentation
|
||||
submenu: Hooks
|
||||
---
|
||||
|
||||
---++ Sqoop Model
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Apache Atlas Hook for Apache Sqoop
|
||||
|
||||
## Sqoop Model
|
||||
Sqoop model includes the following types:
|
||||
* Entity types:
|
||||
* sqoop_process
|
||||
|
|
@ -9,7 +20,6 @@ Sqoop model includes the following types:
|
|||
* sqoop_dbdatastore
|
||||
* super-types: !DataSet
|
||||
* attributes: qualifiedName, name, description, owner, dbStoreType, storeUse, storeUri, source
|
||||
|
||||
* Enum types:
|
||||
* sqoop_operation_type
|
||||
* values: IMPORT, EXPORT, EVAL
|
||||
|
|
@ -17,48 +27,52 @@ Sqoop model includes the following types:
|
|||
* values: TABLE, QUERY, PROCEDURE, OTHER
|
||||
|
||||
Sqoop entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below.
|
||||
<verbatim>
|
||||
sqoop_process.qualifiedName: sqoop <operation> --connect <url> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>]
|
||||
sqoop_dbdatastore.qualifiedName: <storeType> --url <storeUri> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>] --hive-<operation> --hive-database <databaseName> [--hive-table <tableName>] --hive-cluster <clusterName>
|
||||
</verbatim>
|
||||
|
||||
---++ Sqoop Hook
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`sqoop_process.qualifiedName: sqoop <operation> --connect <url> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>]
|
||||
sqoop_dbdatastore.qualifiedName: <storeType> --url <storeUri> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>] --hive-<operation> --hive-database <databaseName> [--hive-table <tableName>] --hive-cluster <clusterName>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
## Sqoop Hook
|
||||
Sqoop added a !SqoopJobDataPublisher that publishes data to Atlas after completion of import Job. Today, only hiveImport is supported in !SqoopHook.
|
||||
This is used to add entities in Atlas using the model detailed above.
|
||||
|
||||
Follow the instructions below to setup Atlas hook in Hive:
|
||||
|
||||
Add the following properties to to enable Atlas hook in Sqoop:
|
||||
* Set-up Atlas hook in <sqoop-conf>/sqoop-site.xml by adding the following:
|
||||
<verbatim>
|
||||
<property>
|
||||
* Set-up Atlas hook in `<sqoop-conf>`/sqoop-site.xml by adding the following:
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`<property>
|
||||
<name>sqoop.job.data.publish.class</name>
|
||||
<value>org.apache.atlas.sqoop.hook.SqoopHook</value>
|
||||
</property></verbatim>
|
||||
</property>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
* untar apache-atlas-${project.version}-sqoop-hook.tar.gz
|
||||
* cd apache-atlas-sqoop-hook-${project.version}
|
||||
* Copy entire contents of folder apache-atlas-sqoop-hook-${project.version}/hook/sqoop to <atlas package>/hook/sqoop
|
||||
* Copy <atlas-conf>/atlas-application.properties to to the sqoop conf directory <sqoop-conf>/
|
||||
* Link <atlas package>/hook/sqoop/*.jar in sqoop lib
|
||||
* Copy entire contents of folder apache-atlas-sqoop-hook-${project.version}/hook/sqoop to `<atlas package>`/hook/sqoop
|
||||
* Copy `<atlas-conf>`/atlas-application.properties to to the sqoop conf directory `<sqoop-conf>`/
|
||||
* Link `<atlas package>`/hook/sqoop/*.jar in sqoop lib
|
||||
|
||||
|
||||
|
||||
The following properties in atlas-application.properties control the thread pool and notification details:
|
||||
<verbatim>
|
||||
atlas.hook.sqoop.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Sqoop operation completion. Default: false
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`atlas.hook.sqoop.synchronous=false # whether to run the hook synchronously. false recommended to avoid delays in Sqoop operation completion. Default: false
|
||||
atlas.hook.sqoop.numRetries=3 # number of retries for notification failure. Default: 3
|
||||
atlas.hook.sqoop.queueSize=10000 # queue size for the threadpool. Default: 10000
|
||||
|
||||
atlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary
|
||||
|
||||
atlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181
|
||||
atlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000
|
||||
atlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000
|
||||
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20
|
||||
</verbatim>
|
||||
atlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [[http://kafka.apache.org/documentation/#producerconfigs][Kafka Producer Configs]]
|
||||
Other configurations for Kafka notification producer can be specified by prefixing the configuration name with "atlas.kafka.". For list of configuration supported by Kafka producer, please refer to [Kafka Producer Configs](http://kafka.apache.org/documentation/#producerconfigs)
|
||||
|
||||
---++ NOTES
|
||||
## NOTES
|
||||
* Only the following sqoop operations are captured by sqoop hook currently
|
||||
* hiveImport
|
||||
|
|
@ -1,6 +1,17 @@
|
|||
---+ Apache Atlas Hook for Apache Storm
|
||||
---
|
||||
name: Storm
|
||||
route: /Hook-Storm
|
||||
menu: Documentation
|
||||
submenu: Hooks
|
||||
---
|
||||
|
||||
---++ Introduction
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Apache Atlas Hook for Apache Storm
|
||||
|
||||
## Introduction
|
||||
|
||||
Apache Storm is a distributed real-time computation system. Storm makes it
|
||||
easy to reliably process unbounded streams of data, doing for real-time
|
||||
|
|
@ -20,7 +31,7 @@ There are 2 parts in this process detailed below:
|
|||
* Storm Atlas Hook to update metadata in Atlas
|
||||
|
||||
|
||||
---++ Storm Data Model
|
||||
## Storm Data Model
|
||||
|
||||
A data model is represented as Types in Atlas. It contains the descriptions
|
||||
of various nodes in the topology graph, such as spouts and bolts and the
|
||||
|
|
@ -39,7 +50,7 @@ if it finds that these are not known to the Atlas server.
|
|||
The data model for each of the types is described in
|
||||
the class definition at org.apache.atlas.storm.model.StormDataModel.
|
||||
|
||||
---++ Storm Atlas Hook
|
||||
## Storm Atlas Hook
|
||||
|
||||
Atlas is notified when a new topology is registered successfully in
|
||||
Storm. Storm provides a hook, backtype.storm.ISubmitterHook, at the Storm client used to
|
||||
|
|
@ -50,7 +61,7 @@ topology and updates Atlas using the types defined. Atlas implements the
|
|||
Storm client hook interface in org.apache.atlas.storm.hook.StormAtlasHook.
|
||||
|
||||
|
||||
---++ Limitations
|
||||
## Limitations
|
||||
|
||||
The following apply for the first version of the integration.
|
||||
|
||||
|
|
@ -59,7 +70,7 @@ The following apply for the first version of the integration.
|
|||
* The Hook currently does not support capturing lineage for custom spouts and bolts.
|
||||
|
||||
|
||||
---++ Installation
|
||||
## Installation
|
||||
|
||||
The Storm Atlas Hook needs to be manually installed in Storm on the client side.
|
||||
* untar apache-atlas-${project.version}-storm-hook.tar.gz
|
||||
|
|
@ -72,16 +83,16 @@ Replace STORM_HOME with storm installation path.
|
|||
Restart all daemons after you have installed the atlas hook into Storm.
|
||||
|
||||
|
||||
---++ Configuration
|
||||
## Configuration
|
||||
|
||||
---+++ Storm Configuration
|
||||
### Storm Configuration
|
||||
|
||||
The Storm Atlas Hook needs to be configured in Storm client config
|
||||
in *$STORM_HOME/conf/storm.yaml* as:
|
||||
|
||||
<verbatim>
|
||||
storm.topology.submission.notifier.plugin.class: "org.apache.atlas.storm.hook.StormAtlasHook"
|
||||
</verbatim>
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`storm.topology.submission.notifier.plugin.class: "org.apache.atlas.storm.hook.StormAtlasHook"`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Also set a 'cluster name' that would be used as a namespace for objects registered in Atlas.
|
||||
This name would be used for namespacing the Storm topology, spouts and bolts.
|
||||
|
|
@ -94,23 +105,23 @@ the client and the cluster name is defined there. This happens similarly for HBa
|
|||
data sets. In case this configuration is not available, the cluster name set in the Storm
|
||||
configuration will be used.
|
||||
|
||||
<verbatim>
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
atlas.cluster.name: "cluster_name"
|
||||
</verbatim>
|
||||
</SyntaxHighlighter>
|
||||
|
||||
In *$STORM_HOME/conf/storm_env.ini*, set an environment variable as follows:
|
||||
|
||||
<verbatim>
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
STORM_JAR_JVM_OPTS:"-Datlas.conf=$ATLAS_HOME/conf/"
|
||||
</verbatim>
|
||||
</SyntaxHighlighter>
|
||||
|
||||
where ATLAS_HOME is pointing to where ATLAS is installed.
|
||||
|
||||
You could also set this up programatically in Storm Config as:
|
||||
|
||||
<verbatim>
|
||||
Config stormConf = new Config();
|
||||
...
|
||||
stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
|
||||
org.apache.atlas.storm.hook.StormAtlasHook.class.getName());
|
||||
</verbatim>
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`Config stormConf = new Config();
|
||||
...
|
||||
stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
|
||||
org.apache.atlas.storm.hook.StormAtlasHook.class.getName());`}
|
||||
</SyntaxHighlighter>
|
||||
|
|
@ -1,12 +1,24 @@
|
|||
---+ Export API
|
||||
---
|
||||
name: Export API
|
||||
route: /Export-API
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Export API
|
||||
The general approach is:
|
||||
* Consumer specifies the scope of data to be exported (details below).
|
||||
* The API if successful, will return the stream in the format specified.
|
||||
* Error will be returned on failure of the call.
|
||||
|
||||
See [[Export-HDFS-API][here]] for details on exporting *hdfs_path* entities.
|
||||
See [here](http://atlas.apache.org/Export-HDFS-API.html) for details on exporting *hdfs_path* entities.
|
||||
|
||||
|*Title*|*Export API*|
|
||||
|**Title**|**Export API**|
|
||||
| ------------ | ------------ |
|
||||
| _Example_ | See Examples sections below. |
|
||||
| _URL_ |_api/atlas/admin/export_ |
|
||||
| _Method_ |_POST_ |
|
||||
|
|
@ -17,13 +29,14 @@ See [[Export-HDFS-API][here]] for details on exporting *hdfs_path* entities.
|
|||
| _Notes_ | Consumer could choose to consume the output of the API by programmatically using _java.io.ByteOutputStream_ or by manually, save the contents of the stream to a file on the disk.|
|
||||
|
||||
__Method Signature__
|
||||
<verbatim>
|
||||
@POST
|
||||
@Path("/export")
|
||||
@Consumes("application/json;charset=UTF-8")
|
||||
</verbatim>
|
||||
|
||||
---+++ Additional Options
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`@POST
|
||||
@Path("/export")
|
||||
@Consumes("application/json;charset=UTF-8")`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
### Additional Options
|
||||
It is possible to specify additional parameters for the _Export_ operation.
|
||||
|
||||
Current implementation has 2 options. Both are optional:
|
||||
|
|
@ -36,17 +49,17 @@ Current implementation has 2 options. Both are optional:
|
|||
* _fetchType_ This option configures the approach used for fetching entities. It has following values:
|
||||
* _FULL_: This fetches all the entities that are connected directly and indirectly to the starting entity. E.g. If a starting entity specified is a table, then this option will fetch the table, database and all the other tables within the database.
|
||||
* _CONNECTED_: This fetches all the etnties that are connected directly to the starting entity. E.g. If a starting entity specified is a table, then this option will fetch the table and the database entity only.
|
||||
* _INCREMENTAL_: See [[Incremental-Export][here]] for details.
|
||||
* _INCREMENTAL_: See [here](http://atlas.apache.org/Incremental-Export.html) for details.
|
||||
|
||||
If no _matchType_ is specified, exact match is used. Which means, that the entire string is used in the search criteria.
|
||||
|
||||
Searching using _matchType_ applies for all types of entities. It is particularly useful for matching entities of type hdfs_path (see [[Export-HDFS-API][here]]).
|
||||
Searching using _matchType_ applies for all types of entities. It is particularly useful for matching entities of type hdfs_path (see (here)[Export-HDFS-API]).
|
||||
|
||||
The _fetchType_ option defaults to _FULL_.
|
||||
|
||||
For complete example see section below.
|
||||
|
||||
---+++ Contents of Exported ZIP File
|
||||
### Contents of Exported ZIP File
|
||||
|
||||
The exported ZIP file has the following entries within it:
|
||||
* _atlas-export-result.json_:
|
||||
|
|
@ -57,20 +70,22 @@ The exported ZIP file has the following entries within it:
|
|||
* _atlas-export-order.json_: Order in which entities should be exported.
|
||||
* _{guid}.json_: Individual entities are exported with file names that correspond to their id.
|
||||
|
||||
---+++ Examples
|
||||
### Examples
|
||||
The _!AtlasExportRequest_ below shows filters that attempt to export 2 databases in cluster cl1:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"itemsToExport": [
|
||||
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "accounts@cl1" } },
|
||||
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "hr@cl1" } }
|
||||
]
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
The _!AtlasExportRequest_ below specifies the _fetchType_ as _FULL_. The _matchType_ option will fetch _accounts@cl1_.
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"itemsToExport": [
|
||||
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "accounts@" } },
|
||||
],
|
||||
|
|
@ -78,12 +93,13 @@ The _!AtlasExportRequest_ below specifies the _fetchType_ as _FULL_. The _matchT
|
|||
"fetchType": "FULL",
|
||||
"matchType": "startsWith"
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
The _!AtlasExportRequest_ below specifies the _fetchType_ as _connected_. The _matchType_ option will fetch _accountsReceivable_, _accountsPayable_, etc present in the database.
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"itemsToExport": [
|
||||
{ "typeName": "hive_db", "uniqueAttributes": { "name": "accounts" } },
|
||||
],
|
||||
|
|
@ -91,15 +107,15 @@ The _!AtlasExportRequest_ below specifies the _fetchType_ as _connected_. The _m
|
|||
"fetchType": "CONNECTED",
|
||||
"matchType": "startsWith"
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Below is the _!AtlasExportResult_ JSON for the export of the _Sales_ DB present in the _!QuickStart_.
|
||||
|
||||
The _metrics_ contains the number of types and entities exported as part of the operation.
|
||||
|
||||
<verbatim>
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"clientIpAddress": "10.0.2.15",
|
||||
"hostName": "10.0.2.2",
|
||||
"metrics": {
|
||||
|
|
@ -128,26 +144,24 @@ The _metrics_ contains the number of types and entities exported as part of the
|
|||
}
|
||||
],
|
||||
"options": {
|
||||
"fetchType": "FULL"
|
||||
"fetchType": "full"
|
||||
}
|
||||
},
|
||||
"userName": "admin"
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
---+++ CURL Calls
|
||||
### CURL Calls
|
||||
Below are sample CURL calls that demonstrate Export of _!QuickStart_ database.
|
||||
|
||||
<verbatim>
|
||||
curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
|
||||
"itemsToExport": [
|
||||
{ "typeName": "DB", "uniqueAttributes": { "name": "Sales" } },
|
||||
{ "typeName": "DB", "uniqueAttributes": { "name": "Reporting" } },
|
||||
{ "typeName": "DB", "uniqueAttributes": { "name": "Logging" } }
|
||||
{ "typeName": "DB", "uniqueAttributes": { "name": "Sales" }
|
||||
{ "typeName": "DB", "uniqueAttributes": { "name": "Reporting" }
|
||||
{ "typeName": "DB", "uniqueAttributes": { "name": "Logging" }
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"fetchType": "FULL"
|
||||
}
|
||||
}' "http://localhost:21000/api/atlas/admin/export" > quickStartDB.zip
|
||||
|
||||
</verbatim>
|
||||
"options": { "full" }
|
||||
}' "http://localhost:21000/api/atlas/admin/export" > quickStartDB.zip`}
|
||||
</SyntaxHighlighter>
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
name: Export HDFS API
|
||||
route: /Export-HDFS-API
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Export & Import APIs for HDFS Path
|
||||
|
||||
### Introduction
|
||||
|
||||
The general approach for using the Import-Export APIs for HDFS Paths remain the same. There are minor variations caused how HDFS paths are handled within Atlas.
|
||||
|
||||
Unlike HIVE entities, HDFS entities within Atlas are created manually using the _Create Entity_ link within the Atlas Web UI.
|
||||
|
||||
Also, HDFS paths tend to be hierarchical, in the sense that users tend to model the same HDFS storage structure within Atlas.
|
||||
|
||||
__Sample HDFS Setup__
|
||||
|
||||
|**HDFS Path**|**Atlas Entity**|
|
||||
| ------------ | ------------ |
|
||||
|<em>/apps/warehouse/finance</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>Finance</em><br/>**QualifiedName: **<em>FinanceAll</em>|
|
||||
|<em>/apps/warehouse/finance/accounts-receivable</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>FinanceReceivable</em><br/>**QualifiedName: **<em>FinanceReceivable</em><br/>**Path: **<em>/apps/warehouse/finance</em>|
|
||||
|<em>/apps/warehouse/finance/accounts-payable</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>Finance-Payable</em><br/>**QualifiedName: **<em>FinancePayable</em><br/>**Path: **<em>/apps/warehouse/finance/accounts-payable</em>|
|
||||
|<em>/apps/warehouse/finance/billing</em>|**Entity type: **<em>hdfs_path</em><br/>**Name: **<em>FinanceBilling</em><br/>**QualifiedName: **<em>FinanceBilling</em><br/>**Path: **<em>/apps/warehouse/finance/billing</em>|
|
||||
|
||||
|
||||
### Export API Using matchType
|
||||
To export entities that represent HDFS path, use the Export API using the _matchType_ option. Details can be found [here](Export-API).
|
||||
|
||||
### Example Using CURL Calls
|
||||
Below are sample CURL calls that performs export operation on the _Sample HDFS Setup_ shown above.
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d '{
|
||||
"itemsToExport": [
|
||||
{ "typeName": "hdfs_path", "uniqueAttributes": { "name": "FinanceAll" }
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"fetchType": "full",
|
||||
"matchType": "startsWith"
|
||||
}
|
||||
}' "http://localhost:21000/api/atlas/admin/export" > financeAll.zip`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
### Automatic Creation of HDFS entities
|
||||
Given that HDFS entity creation is a manual process. The Export API offers a mechanism for creation of requested HDFS entities.
|
||||
|
|
@ -1,20 +1,14 @@
|
|||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one
|
||||
~ or more contributor license agreements. See the NOTICE file
|
||||
~ distributed with this work for additional information
|
||||
~ regarding copyright ownership. The ASF licenses this file
|
||||
~ to you under the Apache License, Version 2.0 (the
|
||||
~ "License"); you may not use this file except in compliance
|
||||
~ with the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
---
|
||||
name: Export Import Audits
|
||||
route: /ExportImportAudits
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Export & Import Audits
|
||||
|
||||
#### Background
|
||||
|
|
@ -23,7 +17,7 @@ The new audits for Export and Import operations also have corresponding REST API
|
|||
|
||||
#### REST APIs
|
||||
|
||||
|Title | Replication Audits for a Cluster |
|
||||
|**Title** | **Replication Audits for a Cluster** |
|
||||
|----------------|------------------------------------------------------------------|
|
||||
|Example | See below. |
|
||||
|URL | api/atlas/admin/expimp/audit |
|
||||
|
|
@ -44,8 +38,8 @@ The new audits for Export and Import operations also have corresponding REST API
|
|||
###### CURL
|
||||
curl -X GET -u admin:admin -H "Content-Type: application/json" -H "Cache-Control: no-cache" 'http://localhost:21000/api/atlas/admin/expimp/audit?sourceClusterName=cl2'
|
||||
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"queryType": "BASIC",
|
||||
"searchParameters": {
|
||||
"typeName": "ReplicationAuditEntry",
|
||||
|
|
@ -89,5 +83,5 @@ curl -X GET -u admin:admin -H "Content-Type: application/json" -H "Cache-Control
|
|||
"displayText": "cl2",
|
||||
"classificationNames": []
|
||||
}]
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
|
@ -1,26 +1,38 @@
|
|||
---+ Import API Options
|
||||
---
|
||||
name: Import API Options
|
||||
route: /Import-API-Options
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Import API Options
|
||||
|
||||
Import API options are specified as _options_ JSON. Since the API accepts multi-part form data, it is possible to sepecify multipls input streams within the CURL call.
|
||||
|
||||
---+++ Examples Using CURL Calls
|
||||
<verbatim>
|
||||
curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
|
||||
### Examples Using CURL Calls
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
|
||||
-H "Cache-Control: no-cache"
|
||||
-F request=@importOptions.json
|
||||
-F data=@quickStartDB.zip
|
||||
"http://localhost:21000/api/atlas/admin/import"
|
||||
</verbatim>
|
||||
"http://localhost:21000/api/atlas/admin/import"`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
To use the defaults, set the contents of _importOptions.json_ to:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"options": {
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
---+++ Options
|
||||
### Options
|
||||
Following options are supported for Import process:
|
||||
|
||||
* Specify transforms during import operation.
|
||||
|
|
@ -28,7 +40,7 @@ Following options are supported for Import process:
|
|||
* Optionally import type definition.
|
||||
* Handling large imports.
|
||||
|
||||
---++++ Transforms
|
||||
#### Transforms
|
||||
|
||||
During the import process, the attribute value of the incoming entity can be changed.
|
||||
|
||||
|
|
@ -43,50 +55,51 @@ Example:
|
|||
The example below applies couple of transforms to the the _qualifiedName_ attribute of hive_table. It converts the value to lower case, then searches for 'cl1', if found, replaces it with 'cl2'.
|
||||
|
||||
To use the option, set the contents of _importOptions.json_ to:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"options": {
|
||||
"transforms": "{ \"hive_table\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] }, \"hive_db\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] } }"
|
||||
"transforms": {"hive_table": { "qualifiedName": [ replace:@cl1:@cl2 ] }, "hive_db": { "qualifiedName": [ replace:@cl1:@cl2 ] } }
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Please refer to [[https://issues.apache.org/jira/browse/ATLAS-1825][ATLAS-1825]] for details scenarios when this option could be used.
|
||||
Please refer to [ATLAS-1825](https://issues.apache.org/jira/browse/ATLAS-1825) for details scenarios when this option could be used.
|
||||
|
||||
---++++ Start Guid or Start Index
|
||||
#### Start Guid or Start Index
|
||||
|
||||
When an import operation is in progress and the server goes down, it would be possible to resume import from the last successfully imported entity. This would allow the import to resume from where it left off.
|
||||
|
||||
Server-side logging is improved to display the detail of the last successfully imported entity, this includes the index within the import list and the entity's guid. Either can be used specify the point to resume import.
|
||||
|
||||
To use the option, set the contents of _importOptions.json_ to:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`{
|
||||
"options": {
|
||||
"startGuid": "bd97c78e-3fa5-4f9c-9f48-3683ca3d1fb1"
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
To use _startPosition_, use the following in the _importOptions.json_:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"options": {
|
||||
"startPosition": "332"
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Steps to use the behavior:
|
||||
* Start an import (using the CURL) that is fairly long, say about 1000+ entities.
|
||||
* Start an import (using the CURL) that is fairly long, say about 1000# entities.
|
||||
* While the import is in progress, stop atlas server (using atlas_stop.py).
|
||||
* From the log file located at _/var/log/atlas/application.log_ get the last successfully imported entity GUID or index position.
|
||||
* Update the _importOptions.json_ with the guid.
|
||||
* Restart import.
|
||||
|
||||
---++++ Optional Importing Type Definition
|
||||
#### Optional Importing Type Definition
|
||||
|
||||
The output of Export has _atlas-typedef.json_ that contains the type definitions for the entities exported.
|
||||
|
||||
|
|
@ -98,44 +111,45 @@ This option allows for optionally importing of type definition. The option is se
|
|||
|
||||
Table below enumerates the conditions that get addressed as part of type definition import:
|
||||
|
||||
|*Condition*|*Action*|
|
||||
|**Condition**|**Action**|
|
||||
| Incoming type does not exist in target system | Type is created. |
|
||||
|Type to be imported and type in target system are same | No change |
|
||||
|Type to be imported and type in target system differ by some attributes| Target system type is updated to the attributes present in the source. It is possible that the target system will have attributes in addition to the one present in the source. In that case, the target system's type attributes will be an union of the attributes. Attributes in target system will not be deleted to match the source. If the type of the attribute differ, import process will be aborted and exception logged.|
|
||||
|
||||
To use the option, set the contents of _importOptions.json_ to:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"options": {
|
||||
"updateTypeDefinition": true
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
---++++ Specifying File to be Imported From Server Location
|
||||
#### Specifying File to be Imported From Server Location
|
||||
|
||||
In scenario where the file to be imported is present at a location on the server, the _importfile_ API can be used. It behaves like the Import API.
|
||||
|
||||
To use the option, set the contents of _importOptions.json_ to:
|
||||
<verbatim>
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"options": {
|
||||
"fileName": "/root/fileToBeImported.zip"
|
||||
}
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
_CURL_
|
||||
<verbatim>
|
||||
curl -g -X POST -u adminuser:password -H "Content-Type: application/json"
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`curl -g -X POST -u adminuser:password -H "Content-Type: application/json"
|
||||
-H "Cache-Control: no-cache"
|
||||
-d r@importOptions.json
|
||||
"http://localhost:21000/api/atlas/admin/importfile"
|
||||
</verbatim>
|
||||
"http://localhost:21000/api/atlas/admin/importfile"`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
---++++ Handling Large Imports
|
||||
#### Handling Large Imports
|
||||
|
||||
By default, the Import Service stores all of the data in memory. This may be limiting for ZIPs containing large amount of data.
|
||||
|
||||
|
|
@ -1,13 +1,26 @@
|
|||
---+ Import API
|
||||
---
|
||||
name: Import API
|
||||
route: /Import-API
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Import API
|
||||
|
||||
The general approach is:
|
||||
* Consumer makes a ZIP file available for import operation. See details below for the 2 flavors of the API.
|
||||
* The API if successful, will return the results of the operation.
|
||||
* Error will be returned on failure of the call.
|
||||
|
||||
---+++ Import ZIP File Using POST
|
||||
### Import ZIP File Using POST
|
||||
|
||||
|*Title*|*Import API*|
|
||||
|
||||
| **Title** | **Import API** |
|
||||
| ------------ | ------------ |
|
||||
| _Example_ | See Examples sections below. |
|
||||
| _Description_|Provide the contents of the file to be imported in the request body.|
|
||||
| _URL_ |_api/atlas/admin/import_ |
|
||||
|
|
@ -17,9 +30,10 @@ The general approach is:
|
|||
| _Success Response_ | _!AtlasImporResult_ is returned as JSON. See details below.|
|
||||
|_Error Response_|Errors that are handled within the system will be returned as _!AtlasBaseException_. |
|
||||
|
||||
---+++ Import ZIP File Available on Server
|
||||
### Import ZIP File Available on Server
|
||||
|
||||
|*Title*|*Import API*|
|
||||
|**Title**|**Import API**|
|
||||
| ------------ | ------------ |
|
||||
| _Example_ | See Examples sections below. |
|
||||
| _Description_|Provide the path of the file to be imported.|
|
||||
| _URL_ |_api/atlas/admin/importfile_ |
|
||||
|
|
@ -31,64 +45,69 @@ The general approach is:
|
|||
|_Notes_| The file to be imported needs to be present on the server at the location specified by the _FILENAME_ parameter.|
|
||||
|
||||
__Method Signature for Import__
|
||||
<verbatim>
|
||||
@POST
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`@POST
|
||||
@Path("/import")
|
||||
@Produces("application/json; charset=UTF-8")
|
||||
@Consumes("multipart/form-data")
|
||||
</verbatim>
|
||||
@Consumes("multipart/form-data")`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
__Method Signature for Import File__
|
||||
<verbatim>
|
||||
@POST
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`@POST
|
||||
@Path("/importfile")
|
||||
@Produces("application/json; charset=UTF-8")
|
||||
@Consumes("application/json")
|
||||
</verbatim>
|
||||
@Consumes("application/json")`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
__Import Options__
|
||||
Please see __[[Import-API-Options][here]]__ for the available options during import process.
|
||||
Please see [here](Import-API-Options) for the available options during import process.
|
||||
|
||||
__!AtlasImportResult Response__
|
||||
The API will return the results of the import operation in the format defined by the _!AtlasImportResult_:
|
||||
* _!AtlasImportParameters_: This contains a collection of name value pair of the options that are applied during the import operation.
|
||||
__AtlasImportResult Response__
|
||||
The API will return the results of the import operation in the format defined by the _AtlasImportResult_:
|
||||
* _AtlasImportParameters_: This contains a collection of name value pair of the options that are applied during the import operation.
|
||||
* _Metrics_: Operation metrics. These include details on the number of types imported, number of entities imported, etc.
|
||||
* _Processed Entities_: Contains list of GUIDs for the entities that were processed.
|
||||
* _Operation Status_: Overall status of the operation. Values are _SUCCESS_, PARTIAL_SUCCESS, _FAIL_.
|
||||
|
||||
---+++ Examples Using CURL Calls
|
||||
### Examples Using CURL Calls
|
||||
The call below performs Import of _!QuickStart_ database using POST.
|
||||
<verbatim>
|
||||
curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
|
||||
-H "Cache-Control: no-cache"
|
||||
-F request=@importOptions.json
|
||||
-F data=@quickStartDB.zip
|
||||
"http://localhost:21000/api/atlas/admin/import"
|
||||
</verbatim>
|
||||
"http://localhost:21000/api/atlas/admin/import"`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
The _request_ parameter is optional. If import has to be run without any options use:
|
||||
<verbatim>
|
||||
curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
|
||||
-H "Cache-Control: no-cache"
|
||||
-F data=@quickStartDB.zip
|
||||
"http://localhost:21000/api/atlas/admin/import"
|
||||
</verbatim>
|
||||
"http://localhost:21000/api/atlas/admin/import"`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
The call below performs Import of _!QuickStart_ database using a ZIP file available on server.
|
||||
<verbatim>
|
||||
curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d ./importOptions.json
|
||||
"http://localhost:21000/api/atlas/admin/importFile" > quickStartDB-import-result.json
|
||||
</verbatim>
|
||||
The call below performs Import of _QuickStart_ database using a ZIP file available on server.
|
||||
|
||||
Below is the _!AtlasImportResult_ JSON for an import that contains _hive_db_.
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d ./importOptions.json
|
||||
"http://localhost:21000/api/atlas/admin/importFile" > quickStartDB-import-result.json`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Below is the _AtlasImportResult_ JSON for an import that contains _hive_db_.
|
||||
|
||||
The _processedEntities_ contains the _guids_ of all the entities imported.
|
||||
|
||||
The _metrics_ contain a breakdown of the types and entities imported along with the operation performed on them viz. _created_ or _updated_.
|
||||
|
||||
<verbatim>
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"request": {
|
||||
"options": {}
|
||||
},
|
||||
|
|
@ -112,12 +131,11 @@ The _metrics_ contain a breakdown of the types and entities imported along with
|
|||
"processedEntities": [
|
||||
"2c4aa713-030b-4fb3-98b1-1cab23d9ac81",
|
||||
"e4aa71ed-70fd-4fa7-9dfb-8250a573e293",
|
||||
|
||||
...
|
||||
|
||||
"ea0f9bdb-1dfc-4e48-9848-a006129929f9",
|
||||
"b5e2cb41-3e7d-4468-84e1-d87c320e75f9"
|
||||
],
|
||||
"operationStatus": "SUCCESS"
|
||||
}
|
||||
</verbatim>
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
---
|
||||
name: Import Export API
|
||||
route: /Import-Export-API
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Export & Import REST APIs
|
||||
|
||||
### What's New
|
||||
The release of 0.8.3 includes the following improvements to Export and Import APIs:
|
||||
* Export: Support for [Incremental Export](Incremental-Export).
|
||||
* Export & Import: Support for [replicated attributes](ReplicatedToFromAttributes) to entities made possible by [SoftReference](SoftReference) entity attribute option.
|
||||
* Export option: [skipLineage](skipLineage).
|
||||
* New entity transforms framework.
|
||||
* New [AtlasServer](AtlasServer) entity type.
|
||||
* Export: [Automatic creation of HDFS path](Export-HDFS-API) requested entities.
|
||||
* New [ExportImportAudits](ExportImportAudits) for Export & Import operations.
|
||||
|
||||
### Background
|
||||
The Import-Export APIs for Atlas facilitate transfer of data to and from a cluster that has Atlas provisioned.
|
||||
|
||||
The APIs when integrated with backup and/or disaster recovery process will ensure participation of Atlas.
|
||||
|
||||
### Introduction
|
||||
There are 2 broad categories viz. Export & Import. The details of the APIs are discussed below.
|
||||
|
||||
The APIs are available only to _admin_ user.
|
||||
|
||||
Only a single import or export operation can be performed at a given time. The operations have a potential for generating large amount. They can also put pressure on resources. This restriction tries to alleviate this problem.
|
||||
|
||||
For Import-Export APIs relating to HDFS path, can be found [here](Import-Export-HDFS-Path).
|
||||
|
||||
For additional information please refer to the following:
|
||||
* [ATLAS-1503](https://issues.apache.org/jira/browse/ATLAS-1503) Original Import-Export API requirements.
|
||||
* [ATLAS-1618](https://issues.apache.org/jira/browse/ATLAS-1618) Export API Scope Specification.
|
||||
|
||||
### Errors
|
||||
If an import or export operation is initiated while another is in progress, the consumer will receive this error:
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`"ATLAS5005E": "Another import or export is in progress. Please try again."`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Unhandled errors will be returned as Internal error code 500.
|
||||
|
||||
### REST API Reference
|
||||
* [Export](Export-API)
|
||||
* [Export HDFS](Export-HDFS-API)
|
||||
* [Import](Import-API)
|
||||
* [Import Options](Import-API-Options)
|
||||
|
||||
|
|
@ -1,3 +1,14 @@
|
|||
---
|
||||
name: Import Entity Transforms
|
||||
route: /ImportEntityTransforms
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# (New) Entity Transforms Framework
|
||||
|
||||
#### Background
|
||||
|
|
@ -30,7 +41,7 @@ The approach used by the new transformation framework creates a transformation b
|
|||
|
||||
Following are built-in conditions.
|
||||
|
||||
Condition Types | Description |
|
||||
|**Condition Types** | **Description** |
|
||||
-----------------------------------------|-----------------|
|
||||
ENTITY_ALL | Any/every entity |
|
||||
ENTITY_TOP_LEVEL | Entity that is the top-level entity. This is also the entity present specified in _AtlasExportRequest_.|
|
||||
|
|
@ -43,7 +54,7 @@ HAS_VALUE | Entity attribute has value. |
|
|||
|
||||
##### Actions
|
||||
|
||||
Action Type | Description |
|
||||
|**Action Type** | *Description** |
|
||||
-------------------|----------------------------------------------|
|
||||
ADD_CLASSIFICATION | Add classifiction |
|
||||
REPLACE_PREFIX | Replace value starting with another value. |
|
||||
|
|
@ -56,87 +67,91 @@ CLEAR | Clear value of an attribute |
|
|||
###### Add Classification
|
||||
|
||||
During import, hive_db entity whose _qualifiedName_ is _stocks@cl1_ will get the classification _clSrcImported_.
|
||||
```json
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"conditions": {
|
||||
"hive_db.qualifiedName": "stocks@cl1"
|
||||
},
|
||||
"action": {
|
||||
"__entity": "ADD_CLASSIFICATION: clSrcImported"
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Every imported entity will get the classification by simply changing the condition. The __entity is special condition which matches entity.
|
||||
|
||||
```json
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"conditions": {
|
||||
"__entity": ""
|
||||
},
|
||||
"action": {
|
||||
"__entity": "ADD_CLASSIFICATION: clSrcImported"
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
To add classification to only the top-level entity (entity that is used as starting point for an export), use:
|
||||
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"conditions": {
|
||||
"__entity": "topLevel:"
|
||||
},
|
||||
"action": {
|
||||
"__entity": "ADD_CLASSIFICATION: clSrcImported"
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
###### Replace Prefix
|
||||
|
||||
This action works on string values. The first parameter is the prefix that is searched for a match, once matched, it is replaced with the provided replacement string.
|
||||
|
||||
The sample below searches for _/aa/bb/_, once found replaces it with _/xx/yy/_.
|
||||
```json
|
||||
{
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"conditions": {
|
||||
"hdfs_path.clusterName": "EQUALS: CL1"
|
||||
},
|
||||
"action": {
|
||||
"hdfs_path.path": "REPLACE_PREFIX: = :/aa/bb/=/xx/yy/"
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
###### To Lower
|
||||
|
||||
Entity whose hdfs_path.clusterName is CL1 will get its path attribute converted to lower case.
|
||||
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"conditions": {
|
||||
"hdfs_path.clusterName": "EQUALS: CL1"
|
||||
},
|
||||
"action": {
|
||||
"hdfs_path.path": "TO_LOWER:"
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
###### Clear
|
||||
|
||||
Entity whose hdfs_path.clusterName has value set, will get its _replicatedTo_ attribute value cleared.
|
||||
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"conditions": {
|
||||
"hdfs_path.clusterName": "HAS_VALUE:"
|
||||
},
|
||||
"action": {
|
||||
"hdfs_path.replicatedTo": "CLEAR:"
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
|
||||
#### Additional Examples
|
||||
|
|
@ -1,3 +1,14 @@
|
|||
---
|
||||
name: Incremental Export
|
||||
route: /Incremental-Export
|
||||
menu: Documentation
|
||||
submenu: Import/Export
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
## Incremental Export
|
||||
|
||||
#### Background
|
||||
|
|
@ -6,8 +17,8 @@ Incremental export allows for export of entities after a specified timestamp. Th
|
|||
#### Export Options
|
||||
New _fetchType_ added to indicate incremental export. This option can be used with any _matchType_. When _fetchType_ is _incremental_, it is necessary to specify the _changeMarker_ option for incremental export to function, else full export will be performed.
|
||||
|
||||
```json
|
||||
{
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{
|
||||
"itemsToExport": [
|
||||
{ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks@cl1" } }
|
||||
],
|
||||
|
|
@ -15,8 +26,8 @@ New _fetchType_ added to indicate incremental export. This option can be used wi
|
|||
"fetchType": "incremental",
|
||||
"changeMarker": 10000
|
||||
}
|
||||
}
|
||||
```
|
||||
}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
#### Getting Change Marker
|
||||
|
||||
71
docs/src/site/twiki/index.twiki → docs/src/documents/Introduction.md
Executable file → Normal file
71
docs/src/site/twiki/index.twiki → docs/src/documents/Introduction.md
Executable file → Normal file
|
|
@ -1,7 +1,14 @@
|
|||
---+ Data Governance and Metadata framework for Hadoop
|
||||
---
|
||||
name: Introduction
|
||||
route: /
|
||||
menu: Introduction
|
||||
---
|
||||
|
||||
# Introduction
|
||||
|
||||
---++ Overview
|
||||
Atlas is Data Governance and Metadata framework for Hadoop
|
||||
|
||||
## Overview
|
||||
|
||||
Atlas is a scalable and extensible set of core foundational governance services – enabling
|
||||
enterprises to effectively and efficiently meet their compliance requirements within Hadoop and
|
||||
|
|
@ -11,79 +18,49 @@ Apache Atlas provides open metadata management and governance capabilities for o
|
|||
to build a catalog of their data assets, classify and govern these assets and provide collaboration
|
||||
capabilities around these data assets for data scientists, analysts and the data governance team.
|
||||
|
||||
---++ Features
|
||||
|
||||
---+++ Metadata types & instances
|
||||
## Features
|
||||
|
||||
### Metadata types & instances
|
||||
* Pre-defined types for various Hadoop and non-Hadoop metadata
|
||||
* Ability to define new types for the metadata to be managed
|
||||
* Types can have primitive attributes, complex attributes, object references; can inherit from other types
|
||||
* Instances of types, called entities, capture metadata object details and their relationships
|
||||
* REST APIs to work with types and instances allow easier integration
|
||||
|
||||
---+++ Classification
|
||||
### Classification
|
||||
* Ability to dynamically create classifications - like PII, EXPIRES_ON, DATA_QUALITY, SENSITIVE
|
||||
* Classifications can include attributes - like expiry_date attribute in EXPIRES_ON classification
|
||||
* Entities can be associated with multiple classifications, enabling easier discovery and security enforcement
|
||||
* Propagation of classifications via lineage - automatically ensures that classifications follow the data as it goes through various processing
|
||||
|
||||
---+++ Lineage
|
||||
### Lineage
|
||||
* Intuitive UI to view lineage of data as it moves through various processes
|
||||
* REST APIs to access and update lineage
|
||||
|
||||
---+++ Search/Discovery
|
||||
### Search/Discovery
|
||||
* Intuitive UI to search entities by type, classification, attribute value or free-text
|
||||
* Rich REST APIs to search by complex criteria
|
||||
* SQL like query language to search entities - Domain Specific Language (DSL)
|
||||
|
||||
---+++ Security & Data Masking
|
||||
### Security & Data Masking
|
||||
* Fine grained security for metadata access, enabling controls on access to entity instances and operations like add/update/remove classifications
|
||||
* Integration with Apache Ranger enables authorization/data-masking on data access based on classifications associated with entities in Apache Atlas. For example:
|
||||
* who can access data classified as PII, SENSITIVE
|
||||
* customer-service users can only see last 4 digits of columns classified as NATIONAL_ID
|
||||
|
||||
|
||||
---++ Getting Started
|
||||
## Getting Started
|
||||
|
||||
* [[WhatsNew-2.0][What's new in Apache Atlas 2.0?]]
|
||||
* [[InstallationSteps][Build & Install]]
|
||||
* [[QuickStart][Quick Start]]
|
||||
* [What's new in Apache Atlas 2.0?](WhatsNew-2.0)
|
||||
* [Build & Install](InstallationSteps)
|
||||
* [Quick Start](QuickStart)
|
||||
|
||||
|
||||
---++ Documentation
|
||||
|
||||
* [[Architecture][High Level Architecture]]
|
||||
* [[TypeSystem][Type System]]
|
||||
* [[Search - Basic][Search: Basic]]
|
||||
* [[Search - Advanced][Search: Advanced]]
|
||||
* [[Glossary][Glossary]]
|
||||
* [[security][Security]]
|
||||
* [[Atlas-Authentication][Authentication]]
|
||||
* [[Atlas-Authorization-Model][Atlas Authorization Model]]
|
||||
* [[Atlas-Authorization-Simple-Authorizer][Steps to configure Atlas Simple Authorizer]]
|
||||
* [[Atlas-Authorization-Ranger-Authorizer][Steps to configure Atlas Ranger Authorizer]]
|
||||
* [[ClassificationPropagation][Classification Propagation]]
|
||||
* [[Configuration][Configuration]]
|
||||
* [[Notifications][Notifications]]
|
||||
* Hooks & Bridges
|
||||
* [[Hook-HBase][HBase Hook & Bridge]]
|
||||
* [[Hook-Hive][Hive Hook & Bridge]]
|
||||
* [[Hook-Sqoop][Sqoop Hook]]
|
||||
* [[Hook-Storm][Storm Hook]]
|
||||
* [[Bridge-Kafka][Kafka Bridge]]
|
||||
* [[HighAvailability][Fault Tolerance And High Availability Options]]
|
||||
* [[Migration-0.8-to-1.0][Migration from Apache Atlas 0.8]]
|
||||
* [[AtlasRepairIndex][Index repair tool]]
|
||||
|
||||
---++ API Documentation
|
||||
## API Documentation
|
||||
|
||||
* <a href="api/v2/index.html">REST API Documentation</a>
|
||||
* [[Import-Export-API][Export & Import REST API Documentation]]
|
||||
* [Export & Import REST API Documentation](Import-Export-API)
|
||||
* <a href="../api/rest.html">Legacy API Documentation</a>
|
||||
|
||||
---++ Developer Setup Documentation
|
||||
* [[EclipseSetup][Developer Setup: Eclipse]]
|
||||
|
||||
#LicenseInfo
|
||||
---+ Licensing Information
|
||||
|
||||
Atlas is distributed under [[http://www.apache.org/licenses/][Apache License 2.0]].
|
||||
## Developer Setup Documentation
|
||||
* [Developer Setup: Eclipse](EclipseSetup)
|
||||
|
|
@ -1,4 +1,15 @@
|
|||
---+ Migrating data from Apache Atlas 0.8 to Apache Atlas 1.0
|
||||
---
|
||||
name: Migration-0.8 to 1.0
|
||||
route: /Migration-0.8-to-1.0
|
||||
menu: Downloads
|
||||
submenu: Migration
|
||||
---
|
||||
|
||||
import themen from 'theme/styles/styled-colors';
|
||||
import * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
|
||||
# Migrating data from Apache Atlas 0.8 to Apache Atlas 1.0
|
||||
|
||||
Apache Atlas 1.0 uses !JanusGraph graph database to store its type and entity details. Prior versions of Apache Atlas
|
||||
use Titan 0.5.4 graph database. The two databases use different formats for storage. For deployments upgrading from
|
||||
|
|
@ -8,14 +19,14 @@ In addition to the change to the graph database, Apache Atlas 1.0 introduces few
|
|||
representation compared to previous versions. Migration steps detailed below will transform data to be compliant with
|
||||
the new internal representation.
|
||||
|
||||
---+++ Migration Steps
|
||||
### Migration Steps
|
||||
|
||||
Migration of data is done in following steps:
|
||||
* Planning the migration.
|
||||
* Export Apache Atlas 0.8 data to a directory on the file system.
|
||||
* Import data from exported files into Apache Atlas 1.0.
|
||||
|
||||
---++++ Planning the migration
|
||||
##### Planning the migration
|
||||
|
||||
The duration of migration of data from Apache Atlas 0.8 to Apache Atlas 1.0 can be significant, depending upon the
|
||||
amount of data present in Apache Atlas. This section helps you to estimate the time to migrate, so that you can plan the
|
||||
|
|
@ -23,17 +34,17 @@ upgrade process better.
|
|||
|
||||
To estimate the time needed to export data, first you need to find the number of entities in Apache Atlas 0.8. This can
|
||||
be done by running the following DSL query:
|
||||
<verbatim>
|
||||
Referenceable select count()
|
||||
</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="sql" style={theme.dark}>
|
||||
{`Referenceable select count()`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Assuming Apache Atlas is deployed in a quad-core CPU with 4 GB of RAM allocated:
|
||||
* Export from Apache Atlas 0.8 will process approximately 2 million entities per hour.
|
||||
* Import into Apache Atlas 1.0 will process approximately 0.7 million entities per hour.
|
||||
|
||||
---++++ Exporting data from Apache Atlas 0.8
|
||||
_Atlas Migration Export Utility_ from Apache Atlas branch-0.8 should be used to export the data from Apache Atlas 0.8
|
||||
deployments. The implementation of which can be found [[https://github.com/apache/atlas/tree/branch-0.8/tools/atlas-migration-exporter][here]].
|
||||
#### Exporting data from Apache Atlas 0.8
|
||||
_Atlas Migration Export Utility_ from Apache Atlas branch-0.8 should be used to export the data from Apache Atlas 0.8 deployments. The implementation of which can be found [here](https://github.com/apache/atlas/tree/branch-0.8/tools/atlas-migration-exporter).
|
||||
|
||||
To build this utility:
|
||||
* Navigate to root directory of Apache Atlas branch-0.8 local repo (say, _/home/atlas/_)
|
||||
|
|
@ -45,44 +56,52 @@ Move the _Atlas Migration Utility_ directory to the Apache Atlas 0.8 cluster.
|
|||
Follow these steps to export the data:
|
||||
* Shutdown _Apache Atlas 0.8_, so that the database is not updated while the migration is in progress.
|
||||
* Execute the following command to export Apache Atlas data in Titan graph database to the specified directory:
|
||||
<verbatim>
|
||||
atlas_migration_export.py -d <output directory></verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`atlas_migration_export.py -d <output directory>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Example:
|
||||
<verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
/home/atlas-migration-utility/atlas_migration_export.py -d /home/atlas-0.8-data
|
||||
</verbatim>
|
||||
</SyntaxHighlighter>
|
||||
|
||||
On successful execution, _Atlas Migration Utility_ tool will display messages like these:
|
||||
<verbatim>
|
||||
atlas-migration-export: starting migration export. Log file location /var/log/atlas/atlas-migration-exporter.log
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`atlas-migration-export: starting migration export. Log file location /var/log/atlas/atlas-migration-exporter.log
|
||||
atlas-migration-export: initializing
|
||||
atlas-migration-export: initialized
|
||||
atlas-migration-export: exporting typesDef to file /home/atlas-0.8-data/atlas-migration-typesdef.json
|
||||
atlas-migration-export: exported typesDef to file /home/atlas-0.8-data/atlas-migration-typesdef.json
|
||||
atlas-migration-export: exporting data to file /home/atlas-0.8-data/atlas-migration-data.json
|
||||
atlas-migration-export: exported data to file /home/atlas-0.8-data/atlas-migration-data.json
|
||||
atlas-migration-export: completed migration export!
|
||||
</verbatim>
|
||||
atlas-migration-export: completed migration export!`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
More details on the progress of export can be found in a log file named _atlas-migration-exporter.log_, in the log directory
|
||||
specified in _atlas-log4j.xml_.
|
||||
|
||||
---+++ Before importing into Apache Atlas 1.0
|
||||
### Before importing into Apache Atlas 1.0
|
||||
* For Apache Atlas deployments that use Solr as index store, please ensure that existing Apache Atlas specific collections are deleted or renamed before installing Apache Atlas 1.0.
|
||||
|
||||
Apache Atlas specific Solr collections can be deleted using CURL commands shown below:
|
||||
<verbatim>
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=DELETE&name=vertex_index'
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl 'http://<solrHost:port>/solr/admin/collections?action=DELETE&name=vertex_index'
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=DELETE&name=edge_index'
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=DELETE&name=fulltext_index'</verbatim>
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=DELETE&name=fulltext_index'`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* Create Solr collections for Apache Atlas 1.0
|
||||
Apache Atlas specific Solr collections can be created using CURL commands shown below:
|
||||
<verbatim>
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=CREATE&name=vertex_index&numShards=1&replicationFactor=1&collection.configName=atlas_configs'
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl 'http://<solrHost:port>/solr/admin/collections?action=CREATE&name=vertex_index&numShards=1&replicationFactor=1&collection.configName=atlas_configs'
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=CREATE&name=edge_index&numShards=1&replicationFactor=1&collection.configName=atlas_configs'
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=CREATE&name=fulltext_index&numShards=1&replicationFactor=1&collection.configName=atlas_configs'</verbatim>
|
||||
curl 'http://<solrHost:port>/solr/admin/collections?action=CREATE&name=fulltext_index&numShards=1&replicationFactor=1&collection.configName=atlas_configs'`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* For Apache Atlas deployments that use HBase as backend store, please note that HBase table used by earlier version can't be used by Apache Atlas 1.0. If you are constrained on disk storage space, the table used by earlier version can be removed after successful export of data.
|
||||
* Apache Atlas 0.8 uses HBase table named 'atlas_titan' (by default)
|
||||
|
|
@ -93,27 +112,33 @@ curl 'http://<solrHost:port>/solr/admin/collections?action=CREATE&name=fulltext_
|
|||
* Make sure the directory containing exported data is accessible to Apache Atlas 1.0 instance.
|
||||
|
||||
|
||||
---++++ Importing Data into Apache Atlas 1.0
|
||||
#### Importing Data into Apache Atlas 1.0
|
||||
Please follow the steps below to import the data exported above into Apache Atlas 1.0:
|
||||
* Specify the location of the directory containing exported data in following property to _atlas-application.properties_:
|
||||
<verbatim>
|
||||
atlas.migration.data.filename=<location of the directory containing exported data></verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`atlas.migration.data.filename=<location of the directory containing exported data>`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
* Start Apache Atlas 1.0. Apache Atlas will start in migration mode. It will start importing data from the specified directory.
|
||||
|
||||
* Monitor the progress of import process with the following curl command:
|
||||
<verbatim>
|
||||
curl -X GET -u admin:<password> -H "Content-Type: application/json" -H "Cache-Control: no-cache" http://<atlasHost>:port/api/atlas/admin/status</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="shell" style={theme.dark}>
|
||||
{`curl -X GET -u admin:<password> -H "Content-Type: application/json" -H "Cache-Control: no-cache" http://<atlasHost>:port/api/atlas/admin/status`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Progress of import will be indicated by a message like this:
|
||||
<verbatim>
|
||||
{"Status":"MIGRATING","MigrationStatus":{"operationStatus":"IN_PROGRESS","startTime":1526512275110,"endTime":1526512302750,"currentIndex":10,"currentCounter":101,"totalCount":0}}
|
||||
</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{"Status":"MIGRATING","MigrationStatus":{"operationStatus":"IN_PROGRESS","startTime":1526512275110,"endTime":1526512302750,"currentIndex":10,"currentCounter":101,"totalCount":0}}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Successful completion of the operation will show a message like this:
|
||||
<verbatim>
|
||||
{"Status":"MIGRATING","MigrationStatus":{"operationStatus":"SUCCESS","startTime":1526512275110,"endTime":1526512302750,"currentIndex":0,"currentCounter":0,"totalCount":371}}
|
||||
</verbatim>
|
||||
|
||||
<SyntaxHighlighter wrapLines={true} language="json" style={theme.dark}>
|
||||
{`{"Status":"MIGRATING","MigrationStatus":{"operationStatus":"SUCCESS","startTime":1526512275110,"endTime":1526512302750,"currentIndex":0,"currentCounter":0,"totalCount":371}}`}
|
||||
</SyntaxHighlighter>
|
||||
|
||||
Once migration import is complete, i.e. _operationStatus_ is _SUCCESS_, follow the steps given below to restart Apache Atlas
|
||||
in _ACTIVE_ mode for regular use:
|
||||
|
|
@ -121,13 +146,13 @@ in _ACTIVE_ mode for regular use:
|
|||
* Remove property _atlas.migration.data.filename_ from _atlas-application.properties_.
|
||||
* Start Apache Atlas 1.0.
|
||||
|
||||
---+++ Atlas Entity Defaults for Migrated Data
|
||||
### Atlas Entity Defaults for Migrated Data
|
||||
|
||||
Apache Atlas 1.0 introduces number of new features. For data that is migrated, the following defaults are set:
|
||||
* All classifications will have _isPropagate_ set to _false_.
|
||||
* Taxonomy terms present in Apache Atlas 0.8, if any, will be converted to classification.
|
||||
|
||||
---++++ Handling of Entity Definitions that use Classifications as Types
|
||||
#### Handling of Entity Definitions that use Classifications as Types
|
||||
|
||||
This features is no longer supported. Classifications that are used as types in _attribute definitions_ (_AttributeDefs_) are converted in to new types whose name has _legacy_ prefix. These are then handled like any other type.
|
||||
Creation of such types was prevented in an earlier release, hence only type definitions have potential to exist. Care has been taken to handle entities of this type as well.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue