1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242import * as commander from 'commander';
import { createReadStream, existsSync, writeFileSync } from 'fs';
import { stringify } from 'csv-stringify';
import { parse } from '@fast-csv/parse';
import { PostHog } from 'posthog-node';
import crypto from 'crypto';
import {
actionRunner,
checkForUpdates,
logRateLimitInformation,
pluralize,
} from '../utils';
import VERSION from '../version';
import { createLogger } from '../logger';
import { createOctokit } from '../octokit';
import { AuditWarning, NameWithOwner } from '../types';
import { auditRepositories } from '../repository-auditor';
import {
MINIMUM_SUPPORTED_GITHUB_ENTERPRISE_SERVER_VERSION,
getGitHubProductInformation,
isSupportedGitHubEnterpriseServerVersion,
} from '../github-products';
import { POSTHOG_API_KEY, POSTHOG_HOST } from '../posthog';
import { createAuthConfig } from '../auth';
const command = new commander.Command();
const { Option } = commander;
interface Arguments {
accessToken?: string;
baseUrl: string;
disableTelemetry: boolean;
inputPath: string;
outputPath: string | undefined;
proxyUrl: string | undefined;
skipUpdateCheck: boolean;
verbose: boolean;
appId?: string | undefined;
privateKey?: string | undefined;
appInstallationId?: string | undefined;
}
const writeWarningsToCsv = async (
warnings: AuditWarning[],
outputPath: string,
): Promise<void> => {
return new Promise((resolve, reject) => {
stringify(
warnings,
{ columns: ['owner', 'name', 'type', 'message'], header: true },
(err, output) => {
if (err) {
reject(err);
} else {
writeFileSync(outputPath, output);
resolve();
}
},
);
});
};
const readNameWithOwnersFromInputFile = async (
inputPath: string,
): Promise<NameWithOwner[]> => {
const nameWithOwners: NameWithOwner[] = [];
return await new Promise((resolve, reject) => {
createReadStream(inputPath, 'utf8')
.pipe(parse({ headers: true }))
.on('error', reject)
.on('data', (row) => {
const rowHeaders = Object.keys(row);
if (
rowHeaders.length === 2 &&
rowHeaders.includes('owner') &&
rowHeaders.includes('name')
) {
if (row.owner) {
const { name, owner } = row;
nameWithOwners.push({ name, owner });
}
} else {
reject(
new Error(
'The input CSV file specified with --input-path is invalid. The file should have a header row with the columns `owner` and `name`, followed by a series of rows.',
),
);
}
})
.on('end', () => {
resolve(nameWithOwners);
});
});
};
command
.name('audit-repos')
.version(VERSION)
.description(
"Audits a list of repos provided in a CSV, identifying data that can't be migrated automatically",
)
.addOption(
new Option('--access-token <access_token>', 'The access token used to interact with the GitHub API. This can also be set using the GITHUB_TOKEN environment variable.')
.env('GITHUB_TOKEN')
)
.addOption(
new Option('--app-installation-id <app_installation_id>', 'The installation ID of the GitHub App.')
.env('GITHUB_APP_INSTALLATION_ID')
)
.addOption(
new Option('--app-id <app_id>', 'The App ID of the GitHub App')
.env('GITHUB_APP_ID')
)
.addOption(
new Option('--private-key <private_key>', 'Content of the *.pem file you downloaded from the about page of the GitHub App. This can also be a path to the *.pem file.')
.env('GITHUB_APP_PRIVATE_KEY')
)
.option(
'--base-url <base_url>',
"The base URL of the GitHub API, if you're running an audit against a GitHub product other than GitHub.com. For GitHub Enterprise Server, this will be something like `https://github.acme.inc/api/v3`. For GitHub Enterprise Cloud with data residency, this will be `https://api.acme.ghe.com`, replacing `acme` with your own tenant.",
'https://api.github.com',
)
.option(
'--output-path <output_path>',
'The path to write the audit result CSV to. Defaults to the "repos" followed by the current date and time, e.g. `repos_1698925405325.csv`.',
)
.requiredOption(
'--input-path <input_path>',
'The path to a input CSV file with a list of repos to audit. The file should have a header row with the columns `owner` and `name`, followed by a series of rows.',
)
.option(
'--proxy-url <proxy_url>',
'The URL of an HTTP(S) proxy to use for requests to the GitHub API (e.g. `http://localhost:3128`). This can also be set using the PROXY_URL environment variable.',
process.env.PROXY_URL,
)
.option('--verbose', 'Whether to emit detailed, verbose logs', false)
.option(
'--disable-telemetry',
'Disable anonymous telemetry that gives the maintainers of this tool basic information about real-world usage. For more detailed information about the built-in telemetry, see the readme at https://github.com/timrogers/gh-migration-audit.',
false,
)
.option('--skip-update-check', 'Skip automatic check for updates to this tool', false)
.action(
actionRunner(async (opts: Arguments) => {
const {
baseUrl,
disableTelemetry,
inputPath,
proxyUrl,
skipUpdateCheck,
verbose,
} = opts;
const logger = createLogger(verbose);
if (!skipUpdateCheck) checkForUpdates(proxyUrl, logger);
const authConfig = createAuthConfig({ ...opts, logger: logger });
const outputPath = opts.outputPath || `repos_${Date.now()}.csv`;
if (existsSync(outputPath)) {
throw new Error(
`The output path, \`${outputPath}\` already exists. Please delete the existing file or specify a different path using the --output-path argument.`,
);
}
if (!existsSync(inputPath)) {
throw new Error(`The input path, \`${inputPath}\` does not exist.`);
}
const octokit = createOctokit(authConfig, baseUrl, proxyUrl, logger);
const shouldCheckRateLimitAgain = await logRateLimitInformation(logger, octokit);
if (shouldCheckRateLimitAgain) {
setInterval(() => {
void logRateLimitInformation(logger, octokit);
}, 30_000);
}
const nameWithOwners = await readNameWithOwnersFromInputFile(inputPath);
if (!nameWithOwners.length) {
throw new Error('The input CSV file does not contain any repos to audit.');
}
logger.info(
`Found ${pluralize(nameWithOwners.length, 'repo', 'repos')} in input CSV file`,
);
const { isGitHubEnterpriseServer, gitHubEnterpriseServerVersion } =
await getGitHubProductInformation(octokit);
if (isGitHubEnterpriseServer) {
if (!isSupportedGitHubEnterpriseServerVersion(gitHubEnterpriseServerVersion)) {
throw new Error(
`GitHub Enterprise Server ${gitHubEnterpriseServerVersion} is not supported. This tool can only be used with GitHub Enterprise Server ${MINIMUM_SUPPORTED_GITHUB_ENTERPRISE_SERVER_VERSION} and later.`,
);
}
logger.info(
`Running in GitHub Enterprise Server ${gitHubEnterpriseServerVersion} mode...`,
);
} else {
logger.info('Running in GitHub.com mode...');
}
const posthog = new PostHog(POSTHOG_API_KEY, {
disabled: disableTelemetry,
host: POSTHOG_HOST,
});
posthog.capture({
distinctId: crypto.randomUUID(),
event: 'audit_repos_start',
properties: {
github_enterprise_server_version: gitHubEnterpriseServerVersion,
is_github_enterprise_server: isGitHubEnterpriseServer,
version: VERSION,
},
});
const warnings = await auditRepositories({
octokit,
logger,
nameWithOwners,
gitHubEnterpriseServerVersion,
});
await writeWarningsToCsv(warnings, outputPath);
logger.info(`Successfully wrote audit CSV to ${outputPath}`);
await posthog.shutdown();
process.exit(0);
}),
);
export default command;