groot/automatisch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: Restructure twitter integration

Browse Source
This commit is contained in:
Faruk AYDIN
2022-10-04 23:09:50 +03:00
parent dc0e03245f
commit 8308265a62
21 changed files with 358 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/backend/src/apps/slack2/auth/verify-credentials.ts
View File

@@ -1,5 +1,5 @@
import qs from 'qs';
import { IGlobalVariableForConnection } from '../../../helpers/global-variable/connection';
import { IGlobalVariableForConnection } from '@automatisch/types';
const verifyCredentials = async ($: IGlobalVariableForConnection) => {
const headers = {

2
packages/backend/src/apps/twitter/triggers/my-tweets.ts
View File

@@ -7,7 +7,7 @@ export default class MyTweets {
this.client = client;
}
async run(lastInternalId: string) {
async run() {
return this.getTweets(lastInternalId);
}

22
packages/backend/src/apps/twitter2/auth/create-auth-data.ts
View File

@@ -1,8 +1,12 @@
import { IJSONObject, IField } from '@automatisch/types';
import oauthClient from '../common/oauth-client';
import generateRequest from '../common/generate-request';
import {
IJSONObject,
IField,
IGlobalVariableForConnection,
} from '@automatisch/types';
import { URLSearchParams } from 'url';
export default async function createAuthData($: any) {
export default async function createAuthData($: IGlobalVariableForConnection) {
try {
const oauthRedirectUrlField = $.app.fields.find(
(field: IField) => field.key == 'oAuthRedirectUrl'
@@ -10,18 +14,10 @@ export default async function createAuthData($: any) {
const callbackUrl = oauthRedirectUrlField.value;
const requestData = {
url: `${$.app.baseUrl}/oauth/request_token`,
const response = await generateRequest($, {
requestPath: '/oauth/request_token',
method: 'POST',
data: { oauth_callback: callbackUrl },
};
const authHeader = oauthClient($).toHeader(
oauthClient($).authorize(requestData)
);
const response = await $.http.post(`/oauth/request_token`, null, {
headers: { ...authHeader },
});
const responseData = Object.fromEntries(new URLSearchParams(response.data));

3
packages/backend/src/apps/twitter2/auth/is-still-verified.ts
View File

@@ -1,6 +1,7 @@
import { IGlobalVariableForConnection } from '@automatisch/types';
import getCurrentUser from '../common/get-current-user';
const isStillVerified = async ($: any) => {
const isStillVerified = async ($: IGlobalVariableForConnection) => {
try {
await getCurrentUser($);
return true;

7
packages/backend/src/apps/twitter2/auth/verify-credentials.ts
View File

@@ -1,7 +1,10 @@
const verifyCredentials = async ($: any) => {
import { IGlobalVariableForConnection } from '@automatisch/types';
import { URLSearchParams } from 'url';
const verifyCredentials = async ($: IGlobalVariableForConnection) => {
try {
const response = await $.http.post(
`/oauth/access_token?oauth_verifier=${$.auth.oauthVerifier}&oauth_token=${$.auth.accessToken}`,
`/oauth/access_token?oauth_verifier=${$.auth.data.oauthVerifier}&oauth_token=${$.auth.data.accessToken}`,
null
);

39
packages/backend/src/apps/twitter2/common/generate-request.ts Normal file
View File

@@ -0,0 +1,39 @@
import { IGlobalVariableForConnection, IJSONObject } from '@automatisch/types';
import oauthClient from './oauth-client';
import { Token } from 'oauth-1.0a';
type IGenereateRequestOptons = {
requestPath: string;
method: string;
data?: IJSONObject;
};
const generateRequest = async (
$: IGlobalVariableForConnection,
options: IGenereateRequestOptons
) => {
const { requestPath, method, data } = options;
const token: Token = {
key: $.auth.data.accessToken as string,
secret: $.auth.data.accessSecret as string,
};
const requestData = {
url: `${$.app.baseUrl}${requestPath}`,
method,
data,
};
const authHeader = oauthClient($).toHeader(
oauthClient($).authorize(requestData, token)
);
const response = await $.http.post(`/oauth/request_token`, null, {
headers: { ...authHeader },
});
return response;
};
export default generateRequest;

14
packages/backend/src/apps/twitter2/common/get-current-user.ts Normal file
View File

@@ -0,0 +1,14 @@
import { IGlobalVariableForConnection } from '@automatisch/types';
import generateRequest from './generate-request';
const getCurrentUser = async ($: IGlobalVariableForConnection) => {
const response = await generateRequest($, {
requestPath: '/2/users/me',
method: 'GET',
});
const currentUser = response.data.data;
return currentUser;
};
export default getCurrentUser;

25
packages/backend/src/apps/twitter2/common/get-user-by-username.ts Normal file
View File

@@ -0,0 +1,25 @@
import { IGlobalVariableForConnection, IJSONObject } from '@automatisch/types';
import generateRequest from './generate-request';
const getUserByUsername = async (
$: IGlobalVariableForConnection,
username: string
) => {
const response = await generateRequest($, {
requestPath: `/2/users/by/username/${username}`,
method: 'GET',
});
if (response.data.errors) {
const errorMessages = response.data.errors
.map((error: IJSONObject) => error.detail)
.join(' ');
throw new Error(`Error occured while fetching user data: ${errorMessages}`);
}
const user = response.data.data;
return user;
};
export default getUserByUsername;

54
packages/backend/src/apps/twitter2/common/get-user-tweets.ts Normal file
View File

@@ -0,0 +1,54 @@
import { IGlobalVariableForConnection, IJSONObject } from '@automatisch/types';
import { URLSearchParams } from 'url';
import omitBy from 'lodash/omitBy';
import isEmpty from 'lodash/isEmpty';
import generateRequest from './generate-request';
const getUserTweets = async (
$: IGlobalVariableForConnection,
userId: string,
lastInternalId?: string
) => {
let response;
const tweets: IJSONObject[] = [];
do {
const params: IJSONObject = {
since_id: lastInternalId,
pagination_token: response?.data?.meta?.next_token,
};
const queryParams = new URLSearchParams(omitBy(params, isEmpty));
const requestPath = `/2/users/${userId}/tweets${
queryParams.toString() ? `?${queryParams.toString()}` : ''
}`;
response = await generateRequest($, {
requestPath,
method: 'GET',
});
if (response.data.meta.result_count > 0) {
response.data.data.forEach((tweet: IJSONObject) => {
if (!lastInternalId || Number(tweet.id) > Number(lastInternalId)) {
tweets.push(tweet);
} else {
return;
}
});
}
} while (response.data.meta.next_token && lastInternalId);
if (response.data.errors) {
const errorMessages = response.data.errors
.map((error: IJSONObject) => error.detail)
.join(' ');
throw new Error(`Error occured while fetching user data: ${errorMessages}`);
}
return tweets;
};
export default getUserTweets;

7
packages/backend/src/apps/twitter2/common/oauth-client/index.ts → packages/backend/src/apps/twitter2/common/oauth-client.ts
View File

@@ -1,10 +1,11 @@
import { IGlobalVariableForConnection } from '@automatisch/types';
import crypto from 'crypto';
import OAuth from 'oauth-1.0a';
const oauthClient = ($: any) => {
const oauthClient = ($: IGlobalVariableForConnection) => {
const consumerData = {
key: $.auth.consumerKey as string,
secret: $.auth.consumerSecret as string,
key: $.auth.data.consumerKey as string,
secret: $.auth.data.consumerSecret as string,
};
return new OAuth({

37
packages/backend/src/apps/twitter2/triggers/my-tweets/index.ts Normal file
View File

@@ -0,0 +1,37 @@
import { IGlobalVariableForConnection } from '@automatisch/types';
import getCurrentUser from '../../common/get-current-user';
import getUserByUsername from '../../common/get-user-by-username';
import getUserTweets from '../../common/get-user-tweets';
export default {
name: 'My Tweets',
key: 'myTweets',
pollInterval: 15,
description: 'Will be triggered when you tweet something new.',
substeps: [
{
key: 'chooseConnection',
name: 'Choose connection',
},
{
key: 'testStep',
name: 'Test trigger',
},
],
async run($: IGlobalVariableForConnection) {
return this.getTweets($, await $.db.flow.lastInternalId());
},
async testRun($: IGlobalVariableForConnection) {
return this.getTweets($);
},
async getTweets($: IGlobalVariableForConnection, lastInternalId?: string) {
const { username } = await getCurrentUser($);
const user = await getUserByUsername($, username);
const tweets = await getUserTweets($, user.id, lastInternalId);
return tweets;
},
};

7
packages/backend/src/helpers/global-variable/connection.ts
View File

@@ -1,5 +1,6 @@
import createHttpClient from '../http-client';
import Connection from '../../models/connection';
import Flow from '../../models/flow';
import {
IJSONObject,
IApp,
@@ -8,7 +9,8 @@ import {
const prepareGlobalVariableForConnection = (
connection: Connection,
appData: IApp
appData: IApp,
flow?: Flow
): IGlobalVariableForConnection => {
return {
auth: {
@@ -24,6 +26,9 @@ const prepareGlobalVariableForConnection = (
},
app: appData,
http: createHttpClient({ baseURL: appData.baseUrl }),
db: {
flow: flow,
},
};
};

9
packages/backend/src/models/execution-step.ts
View File

@@ -3,14 +3,15 @@ import Base from './base';
import Execution from './execution';
import Step from './step';
import Telemetry from '../helpers/telemetry';
import { IJSONObject } from '@automatisch/types';
class ExecutionStep extends Base {
id!: string;
executionId!: string;
stepId!: string;
dataIn!: Record<string, unknown>;
dataOut!: Record<string, unknown>;
errorDetails: Record<string, unknown>;
dataIn!: IJSONObject;
dataOut!: IJSONObject;
errorDetails: IJSONObject;
status = 'failure';
step: Step;
@@ -23,7 +24,7 @@ class ExecutionStep extends Base {
id: { type: 'string', format: 'uuid' },
executionId: { type: 'string', format: 'uuid' },
stepId: { type: 'string' },
dataIn: { type: 'object' },
dataIn: { type: ['object', 'null'] },
dataOut: { type: ['object', 'null'] },
status: { type: 'string', enum: ['success', 'failure'] },
errorDetails: { type: ['object', 'null'] },

2
packages/backend/src/models/flow.ts
View File

@@ -10,7 +10,7 @@ class Flow extends Base {
name!: string;
userId!: string;
active: boolean;
steps?: [Step];
steps: Step[];
published_at: string;
static tableName = 'flows';

2
packages/backend/src/models/step.ts
View File

@@ -20,7 +20,7 @@ class Step extends Base {
parameters: Record<string, unknown>;
connection?: Connection;
flow: Flow;
executionSteps?: [ExecutionStep];
executionSteps: ExecutionStep[];
static tableName = 'steps';

6
packages/backend/src/models/user.ts
View File

@@ -10,9 +10,9 @@ class User extends Base {
id!: string;
email!: string;
password!: string;
connections?: [Connection];
flows?: [Flow];
steps?: [Step];
connections?: Connection[];
flows?: Flow[];
steps?: Step[];
static tableName = 'users';

4
packages/backend/src/services/processor.ts
View File

@@ -112,8 +112,8 @@ class Processor {
await execution.$relatedQuery('executionSteps').insertAndFetch({
stepId: id,
status: 'failure',
dataIn: computedParameters,
dataOut: null,
dataIn: null,
dataOut: computedParameters,
errorDetails: fetchedActionData.error,
});

15
packages/types/index.d.ts vendored
View File

@@ -47,21 +47,21 @@ export interface IExecution {
export interface IStep {
id: string;
name: string;
name?: string;
flowId: string;
key: string;
appKey: string;
key?: string;
appKey?: string;
iconUrl: string;
type: 'action' | 'trigger';
connectionId: string;
connectionId?: string;
status: string;
position: number;
parameters: Record<string, unknown>;
connection: Partial<IConnection>;
connection?: Partial<IConnection>;
flow: IFlow;
executionSteps: IExecutionStep[];
// FIXME: remove this property once execution steps are properly exposed via queries
output: IJSONObject;
output?: IJSONObject;
appData?: IApp;
}
@@ -202,6 +202,9 @@ export type IGlobalVariableForConnection = {
};
app: IApp;
http: IHttpClient;
db: {
flow: IFlow;
};
};
declare module 'axios' {

75
packages/web/src/components/ChooseAppAndEventSubstep/index.tsx
View File

@@ -15,7 +15,7 @@ import FlowSubstepTitle from 'components/FlowSubstepTitle';
import type { IApp, IStep, ISubstep } from '@automatisch/types';
type ChooseAppAndEventSubstepProps = {
substep: ISubstep,
substep: ISubstep;
expanded?: boolean;
onExpand: () => void;
onCollapse: () => void;
@@ -24,14 +24,17 @@ type ChooseAppAndEventSubstepProps = {
step: IStep;
};
const optionGenerator = (app: IApp): { label: string; value: string; } => ({
const optionGenerator = (app: IApp): { label: string; value: string } => ({
label: app.name as string,
value: app.key as string,
});
const getOption = (options: Record<string, unknown>[], appKey: IApp["key"]) => options.find(option => option.value === appKey as string) || null;
const getOption = (options: Record<string, unknown>[], appKey?: IApp['key']) =>
options.find((option) => option.value === (appKey as string)) || null;
function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.ReactElement {
function ChooseAppAndEventSubstep(
props: ChooseAppAndEventSubstepProps
): React.ReactElement {
const {
substep,
expanded = false,
@@ -47,23 +50,33 @@ function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.R
const isTrigger = step.type === 'trigger';
const { data } = useQuery(GET_APPS, { variables: { onlyWithTriggers: isTrigger }});
const { data } = useQuery(GET_APPS, {
variables: { onlyWithTriggers: isTrigger },
});
const apps: IApp[] = data?.getApps;
const app = apps?.find((currentApp: IApp) => currentApp.key === step.appKey);
const appOptions = React.useMemo(() => apps?.map((app) => optionGenerator(app)), [apps]);
const appOptions = React.useMemo(
() => apps?.map((app) => optionGenerator(app)),
[apps]
);
const actionsOrTriggers = isTrigger ? app?.triggers : app?.actions;
const actionOptions = React.useMemo(() => actionsOrTriggers?.map((trigger) => optionGenerator(trigger)) ?? [], [app?.key]);
const selectedActionOrTrigger = actionsOrTriggers?.find((actionOrTrigger) => actionOrTrigger.key === step?.key) || null;
const actionOptions = React.useMemo(
() => actionsOrTriggers?.map((trigger) => optionGenerator(trigger)) ?? [],
[app?.key]
);
const selectedActionOrTrigger =
actionsOrTriggers?.find(
(actionOrTrigger) => actionOrTrigger.key === step?.key
) || null;
const {
name,
} = substep;
const { name } = substep;
const valid: boolean = !!step.key && !!step.appKey;
// placeholders
const onEventChange = React.useCallback((event: React.SyntheticEvent, selectedOption: unknown) => {
const onEventChange = React.useCallback(
(event: React.SyntheticEvent, selectedOption: unknown) => {
if (typeof selectedOption === 'object') {
// TODO: try to simplify type casting below.
const typedSelectedOption = selectedOption as { value: string };
@@ -79,9 +92,12 @@ function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.R
});
}
}
}, [step, onChange]);
},
[step, onChange]
);
const onAppChange = React.useCallback((event: React.SyntheticEvent, selectedOption: unknown) => {
const onAppChange = React.useCallback(
(event: React.SyntheticEvent, selectedOption: unknown) => {
if (typeof selectedOption === 'object') {
// TODO: try to simplify type casting below.
const typedSelectedOption = selectedOption as { value: string };
@@ -99,7 +115,9 @@ function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.R
});
}
}
}, [step, onChange]);
},
[step, onChange]
);
const onToggle = expanded ? onCollapse : onExpand;
@@ -112,14 +130,26 @@ function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.R
valid={valid}
/>
<Collapse in={expanded} timeout="auto" unmountOnExit>
<ListItem sx={{ pt: 2, pb: 3, flexDirection: 'column', alignItems: 'flex-start' }}>
<ListItem
sx={{
pt: 2,
pb: 3,
flexDirection: 'column',
alignItems: 'flex-start',
}}
>
<Autocomplete
fullWidth
disablePortal
disableClearable
disabled={editorContext.readOnly}
options={appOptions}
renderInput={(params) => <TextField {...params} label={formatMessage('flowEditor.chooseApp')} />}
renderInput={(params) => (
<TextField
{...params}
label={formatMessage('flowEditor.chooseApp')}
/>
)}
value={getOption(appOptions, step.appKey)}
onChange={onAppChange}
/>
@@ -137,7 +167,12 @@ function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.R
disableClearable
disabled={editorContext.readOnly}
options={actionOptions}
renderInput={(params) => <TextField {...params} label={formatMessage('flowEditor.chooseEvent')} />}
renderInput={(params) => (
<TextField
{...params}
label={formatMessage('flowEditor.chooseEvent')}
/>
)}
value={getOption(actionOptions, step.key)}
onChange={onEventChange}
/>
@@ -147,7 +182,9 @@ function ChooseAppAndEventSubstep(props: ChooseAppAndEventSubstepProps): React.R
{isTrigger && selectedActionOrTrigger?.pollInterval && (
<TextField
label={formatMessage('flowEditor.pollIntervalLabel')}
value={formatMessage('flowEditor.pollIntervalValue', { minutes: selectedActionOrTrigger.pollInterval })}
value={formatMessage('flowEditor.pollIntervalValue', {
minutes: selectedActionOrTrigger.pollInterval,
})}
sx={{ mt: 2 }}
fullWidth
disabled

71
packages/web/src/components/PowerInput/Suggestions.tsx
View File

@@ -24,13 +24,10 @@ const LIST_HEIGHT = 256;
const getPartialArray = (array: any[], length = array.length) => {
return array.slice(0, length);
}
};
const Suggestions = (props: SuggestionsProps) => {
const {
data,
onSuggestionClick = () => null,
} = props;
const { data, onSuggestionClick = () => null } = props;
const [current, setCurrent] = React.useState<number | null>(0);
const [listLength, setListLength] = React.useState<number>(SHORT_LIST_LENGTH);
@@ -40,41 +37,43 @@ const Suggestions = (props: SuggestionsProps) => {
const collapseList = () => {
setListLength(SHORT_LIST_LENGTH);
}
};
React.useEffect(() => {
setListLength(SHORT_LIST_LENGTH);
}, [current])
}, [current]);
return (
<Paper
elevation={5}
sx={{ width: '100%' }}
>
<Typography variant="subtitle2" sx={{ p: 2, }}>Variables</Typography>
<List
disablePadding
>
<Paper elevation={5} sx={{ width: '100%' }}>
<Typography variant="subtitle2" sx={{ p: 2 }}>
Variables
</Typography>
<List disablePadding>
{data.map((option: IStep, index: number) => (
<>
<ListItemButton
divider
onClick={() => setCurrent((currentIndex) => currentIndex === index ? null : index)}
sx={{ py: 0.5, }}
onClick={() =>
setCurrent((currentIndex) =>
currentIndex === index ? null : index
)
}
sx={{ py: 0.5 }}
>
<ListItemText
primary={option.name}
/>
<ListItemText primary={option.name} />
{!!option.output?.length && (
current === index ? <ExpandLess /> : <ExpandMore />
)}
{!!option.output?.length &&
(current === index ? <ExpandLess /> : <ExpandMore />)}
</ListItemButton>
<Collapse in={current === index} timeout="auto" unmountOnExit>
<List component="div" disablePadding sx={{ maxHeight: LIST_HEIGHT, overflowY: 'auto' }}>
{getPartialArray(option.output as any || [], listLength)
.map((suboption: any, index: number) => (
<List
component="div"
disablePadding
sx={{ maxHeight: LIST_HEIGHT, overflowY: 'auto' }}
>
{getPartialArray((option.output as any) || [], listLength).map(
(suboption: any, index: number) => (
<ListItemButton
sx={{ pl: 4 }}
divider
@@ -85,7 +84,7 @@ const Suggestions = (props: SuggestionsProps) => {
primaryTypographyProps={{
variant: 'subtitle1',
title: 'Property name',
sx: { fontWeight: 700 }
sx: { fontWeight: 700 },
}}
secondary={suboption.value || ''}
secondaryTypographyProps={{
@@ -95,24 +94,18 @@ const Suggestions = (props: SuggestionsProps) => {
}}
/>
</ListItemButton>
))
}
)
)}
</List>
{option.output?.length > listLength && (
<Button
fullWidth
onClick={expandList}
>
{(option.output?.length || 0) > listLength && (
<Button fullWidth onClick={expandList}>
Show all
</Button>
)}
{listLength === Infinity && (
<Button
fullWidth
onClick={collapseList}
>
<Button fullWidth onClick={collapseList}>
Show less
</Button>
)}
@@ -122,6 +115,6 @@ const Suggestions = (props: SuggestionsProps) => {
</List>
</Paper>
);
}
};
export default Suggestions;

31
packages/web/src/components/PowerInput/data.ts
View File

@@ -1,6 +1,7 @@
import type { IStep } from '@automatisch/types';
const joinBy = (delimiter = '.', ...args: string[]) => args.filter(Boolean).join(delimiter);
const joinBy = (delimiter = '.', ...args: string[]) =>
args.filter(Boolean).join(delimiter);
const process = (data: any, parentKey?: any, index?: number): any[] => {
if (typeof data !== 'object') {
@@ -8,14 +9,19 @@ const process = (data: any, parentKey?: any, index?: number): any[] => {
{
name: `${parentKey}.${index}`,
value: data,
}
]
},
];
}
const entries = Object.entries(data);
return entries.flatMap(([name, value]) => {
const fullName = joinBy('.', parentKey, (index as number)?.toString(), name);
const fullName = joinBy(
'.',
parentKey,
(index as number)?.toString(),
name
);
if (Array.isArray(value)) {
return value.flatMap((item, index) => process(item, fullName, index));
@@ -25,10 +31,12 @@ const process = (data: any, parentKey?: any, index?: number): any[] => {
return process(value, fullName);
}
return [{
return [
{
name: fullName,
value,
}];
},
];
});
};
@@ -39,12 +47,17 @@ export const processStepWithExecutions = (steps: IStep[]): any[] => {
.filter((step: IStep) => {
const hasExecutionSteps = !!step.executionSteps?.length;
return hasExecutionSteps
return hasExecutionSteps;
})
.map((step: IStep, index: number) => ({
id: step.id,
// TODO: replace with step.name once introduced
name: `${index + 1}. ${step.appKey?.charAt(0)?.toUpperCase() + step.appKey?.slice(1)}`,
output: process(step.executionSteps?.[0]?.dataOut || {}, `step.${step.id}`),
name: `${index + 1}. ${
(step.appKey || '').charAt(0)?.toUpperCase() + step.appKey?.slice(1)
}`,
output: process(
step.executionSteps?.[0]?.dataOut || {},
`step.${step.id}`
),
}));
};