Page Menu
Home
Sealhub
Search
Configure Global Search
Log In
Files
F9583953
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
27 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/scripts/lib/PhutilLibraryMapBuilder.php b/scripts/lib/PhutilLibraryMapBuilder.php
index 45fbdd01..3b0f31eb 100755
--- a/scripts/lib/PhutilLibraryMapBuilder.php
+++ b/scripts/lib/PhutilLibraryMapBuilder.php
@@ -1,505 +1,505 @@
<?php
/**
* Build maps of libphutil libraries. libphutil uses the library map to locate
* and load classes and functions in the library.
*
* @task map Mapping libphutil Libraries
* @task path Path Management
* @task symbol Symbol Analysis and Caching
* @task source Source Management
*/
final class PhutilLibraryMapBuilder {
private $root;
private $quiet;
private $subprocessLimit = 8;
private $ugly;
private $showMap;
const LIBRARY_MAP_VERSION_KEY = '__library_version__';
const LIBRARY_MAP_VERSION = 2;
const SYMBOL_CACHE_VERSION_KEY = '__symbol_cache_version__';
const SYMBOL_CACHE_VERSION = 8;
/* -( Mapping libphutil Libraries )---------------------------------------- */
/**
* Create a new map builder for a library.
*
* @param string Path to the library root.
*
* @task map
*/
public function __construct($root) {
$this->root = $root;
}
/**
* Control status output. Use --quiet to set this.
*
* @param bool If true, don't show status output.
* @return this
*
* @task map
*/
public function setQuiet($quiet) {
$this->quiet = $quiet;
return $this;
}
/**
* Control subprocess parallelism limit. Use --limit to set this.
*
* @param int Maximum number of subprocesses to run in parallel.
* @return this
*
* @task map
*/
public function setSubprocessLimit($limit) {
$this->subprocessLimit = $limit;
return $this;
}
/**
* Control whether the ugly (but fast) or pretty (but slower) JSON formatter
* is used.
*
* @param bool If true, use the fastest formatter.
* @return this
*
* @task map
*/
public function setUgly($ugly) {
$this->ugly = $ugly;
return $this;
}
/**
* Control whether the map should be rebuilt, or just shown (printed to
* stdout in JSON).
*
* @param bool If true, show map instead of updating.
* @return this
*
* @task map
*/
public function setShowMap($show_map) {
$this->showMap = $show_map;
return $this;
}
/**
* Build or rebuild the library map.
*
* @return this
*
* @task map
*/
public function buildMap() {
// Identify all the ".php" source files in the library.
$this->log("Finding source files...\n");
$source_map = $this->loadSourceFileMap();
$this->log("Found ".number_format(count($source_map))." files.\n");
// Load the symbol cache with existing parsed symbols. This allows us
// to remap libraries quickly by analyzing only changed files.
$this->log("Loading symbol cache...\n");
$symbol_cache = $this->loadSymbolCache();
// Build out the symbol analysis for all the files in the library. For
// each file, check if it's in cache. If we miss in the cache, do a fresh
// analysis.
$symbol_map = array();
$futures = array();
foreach ($source_map as $file => $hash) {
if (!empty($symbol_cache[$hash])) {
$symbol_map[$file] = $symbol_cache[$hash];
continue;
}
$futures[$file] = $this->buildSymbolAnalysisFuture($file);
}
$this->log("Found ".number_format(count($symbol_map))." files in cache.\n");
// Run the analyzer on any files which need analysis.
if ($futures) {
$limit = $this->subprocessLimit;
$count = number_format(count($futures));
$this->log("Analyzing {$count} files with {$limit} subprocesses...\n");
foreach (Futures($futures)->limit($limit) as $file => $future) {
$result = $future->resolveJSON();
if (empty($result['error'])) {
$symbol_map[$file] = $result;
} else {
echo phutil_console_format(
"\n**SYNTAX ERROR!**\nFile: %s\nLine: %d\n\n%s\n",
Filesystem::readablePath($result['file']),
$result['line'],
$result['error']);
exit(1);
}
$this->log(".");
}
$this->log("\nDone.\n");
}
// We're done building the cache, so write it out immediately. Note that
// we've only retained entries for files we found, so this implicitly cleans
// out old cache entries.
$this->writeSymbolCache($symbol_map, $source_map);
// Our map is up to date, so either show it on stdout or write it to disk.
if ($this->showMap) {
$this->log("Showing map...\n");
if ($this->ugly) {
echo json_encode($symbol_map);
} else {
$json = new PhutilJSON();
echo $json->encodeFormatted($symbol_map);
}
} else {
$this->log("Building library map...\n");
$library_map = $this->buildLibraryMap($symbol_map);
$this->log("Writing map...\n");
$this->writeLibraryMap($library_map);
}
$this->log("Done.\n");
return $this;
}
/**
* Write a status message to the user, if not running in quiet mode.
*
* @param string Message to write.
* @return this
*
* @task map
*/
private function log($message) {
if (!$this->quiet) {
@fwrite(STDERR, $message);
}
return $this;
}
/* -( Path Management )---------------------------------------------------- */
/**
* Get the path to some file in the library.
*
* @param string A library-relative path. If omitted, returns the library
* root path.
* @return string An absolute path.
*
* @task path
*/
private function getPath($path = '') {
return $this->root.'/'.$path;
}
/**
* Get the path to the symbol cache file.
*
* @return string Absolute path to symbol cache.
*
* @task path
*/
private function getPathForSymbolCache() {
return $this->getPath('.phutil_module_cache');
}
/**
* Get the path to the map file.
*
* @return string Absolute path to the library map.
*
* @task path
*/
private function getPathForLibraryMap() {
return $this->getPath('__phutil_library_map__.php');
}
/**
* Get the path to the library init file.
*
* @return string Absolute path to the library init file
*
* @task path
*/
private function getPathForLibraryInit() {
return $this->getPath('__phutil_library_init__.php');
}
/* -( Symbol Analysis and Caching )---------------------------------------- */
/**
* Load the library symbol cache, if it exists and is readable and valid.
*
* @return dict Map of content hashes to cache of output from
* `phutil_symbols.php`.
*
* @task symbol
*/
private function loadSymbolCache() {
$cache_file = $this->getPathForSymbolCache();
try {
$cache = Filesystem::readFile($cache_file);
} catch (Exception $ex) {
$cache = null;
}
$symbol_cache = array();
if ($cache) {
$symbol_cache = json_decode($cache, true);
if (!is_array($symbol_cache)) {
$symbol_cache = array();
}
}
$version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
if ($version != self::SYMBOL_CACHE_VERSION) {
// Throw away caches from a different version of the library.
$symbol_cache = array();
}
unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);
return $symbol_cache;
}
/**
* Write a symbol map to disk cache.
*
* @param dict Symbol map of relative paths to symbols.
* @param dict Source map (like @{method:loadSourceFileMap}).
* @return void
*
* @task symbol
*/
private function writeSymbolCache(array $symbol_map, array $source_map) {
$cache_file = $this->getPathForSymbolCache();
$cache = array(
self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
);
foreach ($symbol_map as $file => $symbols) {
$cache[$source_map[$file]] = $symbols;
}
$json = json_encode($cache);
try {
Filesystem::writeFile($cache_file, $json);
} catch (FilesystemException $ex) {
$this->log("Unable to save the cache!\n");
}
}
/**
* Drop the symbol cache, forcing a clean rebuild.
*
* @return this
*
* @task symbol
*/
public function dropSymbolCache() {
$this->log("Dropping symbol cache...\n");
Filesystem::remove($this->getPathForSymbolCache());
}
/**
* Build a future which returns a `phutil_symbols.php` analysis of a source
* file.
*
* @param string Relative path to the source file to analyze.
* @return Future Analysis future.
*
* @task symbol
*/
private function buildSymbolAnalysisFuture($file) {
$absolute_file = $this->getPath($file);
$bin = dirname(dirname(__FILE__)).'/phutil_symbols.php';
- return new ExecFuture('%s --ugly -- %s', $bin, $absolute_file);
+ return new ExecFuture('php %s --ugly -- %s', $bin, $absolute_file);
}
/* -( Source Management )-------------------------------------------------- */
/**
* Build a map of all source files in a library to hashes of their content.
* Returns an array like this:
*
* array(
* 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
* // ...
* );
*
* @return dict Map of library-relative paths to content hashes.
* @task source
*/
private function loadSourceFileMap() {
$root = $this->getPath();
$init = $this->getPathForLibraryInit();
if (!Filesystem::pathExists($init)) {
throw new Exception("Provided path '{$root}' is not a phutil library.");
}
$files = id(new FileFinder($root))
->withType('f')
->withSuffix('php')
->excludePath('*/.*')
->setGenerateChecksums(true)
->find();
$map = array();
foreach ($files as $file => $hash) {
if (basename($file) == '__init__.php') {
// TODO: Remove this once we kill __init__.php. This just makes the
// script run faster until we do, so testing and development is less
// annoying.
continue;
}
$file = Filesystem::readablePath($file, $root);
$file = ltrim($file, '/');
if (dirname($file) == '.') {
// We don't permit normal source files at the root level, so just ignore
// them; they're special library files.
continue;
}
// We include also filename in the hash to handle cases when the file is
// moved without modifying its content.
$map[$file] = md5($hash.$file);
}
return $map;
}
/**
* Convert the symbol analysis of all the source files in the library into
* a library map.
*
* @param dict Symbol analysis of all source files.
* @return dict Library map.
* @task source
*/
private function buildLibraryMap(array $symbol_map) {
$library_map = array(
'class' => array(),
'function' => array(),
'xmap' => array(),
);
// Detect duplicate symbols within the library.
foreach ($symbol_map as $file => $info) {
foreach ($info['have'] as $type => $symbols) {
foreach ($symbols as $symbol => $declaration) {
$lib_type = ($type == 'interface') ? 'class' : $type;
if (!empty($library_map[$lib_type][$symbol])) {
$prior = $library_map[$lib_type][$symbol];
throw new Exception(
"Definition of {$type} '{$symbol}' in file '{$file}' duplicates ".
"prior definition in file '{$prior}'. You can not declare the ".
"same symbol twice.");
}
$library_map[$lib_type][$symbol] = $file;
}
}
$library_map['xmap'] += $info['xmap'];
}
// Simplify the common case (one parent) to make the file a little easier
// to deal with.
foreach ($library_map['xmap'] as $class => $extends) {
if (count($extends) == 1) {
$library_map['xmap'][$class] = reset($extends);
}
}
// Sort the map so it is relatively stable across changes.
foreach ($library_map as $key => $symbols) {
ksort($symbols);
$library_map[$key] = $symbols;
}
ksort($library_map);
return $library_map;
}
/**
* Write a finalized library map.
*
* @param dict Library map structure to write.
* @return void
*
* @task source
*/
private function writeLibraryMap(array $library_map) {
$map_file = $this->getPathForLibraryMap();
$version = self::LIBRARY_MAP_VERSION;
$library_map = array(
self::LIBRARY_MAP_VERSION_KEY => $version,
) + $library_map;
$library_map = var_export($library_map, $return_string = true);
$library_map = preg_replace('/\s+$/m', '', $library_map);
$library_map = preg_replace('/array \(/', 'array(', $library_map);
$at = '@';
$source_file = <<<EOPHP
<?php
/**
* This file is automatically generated. Use 'arc liberate' to rebuild it.
* {$at}generated
* {$at}phutil-library-version {$version}
*/
phutil_register_library_map({$library_map});
EOPHP;
Filesystem::writeFile($map_file, $source_file);
}
}
diff --git a/src/lint/linter/ArcanistPhutilLibraryLinter.php b/src/lint/linter/ArcanistPhutilLibraryLinter.php
index 846e5467..0a8f772e 100644
--- a/src/lint/linter/ArcanistPhutilLibraryLinter.php
+++ b/src/lint/linter/ArcanistPhutilLibraryLinter.php
@@ -1,181 +1,181 @@
<?php
/**
* Applies lint rules for Phutil libraries. We enforce three rules:
*
* # If you use a symbol, it must be defined somewhere.
* # If you define a symbol, it must not duplicate another definition.
* # If you define a class or interface in a file, it MUST be the only symbol
* defined in that file.
*
* @group linter
*/
final class ArcanistPhutilLibraryLinter extends ArcanistLinter {
const LINT_UNKNOWN_SYMBOL = 1;
const LINT_DUPLICATE_SYMBOL = 2;
const LINT_ONE_CLASS_PER_FILE = 3;
public function getLintNameMap() {
return array(
self::LINT_UNKNOWN_SYMBOL => 'Unknown Symbol',
self::LINT_DUPLICATE_SYMBOL => 'Duplicate Symbol',
self::LINT_ONE_CLASS_PER_FILE => 'One Class Per File',
);
}
public function getLinterName() {
return 'PHL';
}
public function getLintSeverityMap() {
return array();
}
public function willLintPaths(array $paths) {
if (!xhpast_is_available()) {
throw new Exception(xhpast_get_build_instructions());
}
// NOTE: For now, we completely ignore paths and just lint every library in
// its entirety. This is simpler and relatively fast because we don't do any
// detailed checks and all the data we need for this comes out of module
// caches.
$bootloader = PhutilBootloader::getInstance();
$libs = $bootloader->getAllLibraries();
// Load the up-to-date map for each library, without loading the library
// itself. This means lint results will accurately reflect the state of
// the working copy.
$arc_root = dirname(phutil_get_library_root('arcanist'));
$bin = "{$arc_root}/scripts/phutil_rebuild_map.php";
$symbols = array();
foreach ($libs as $lib) {
// Do these one at a time since they individually fanout to saturate
// available system resources.
$future = new ExecFuture(
- '%s --show --quiet --ugly -- %s',
+ 'php %s --show --quiet --ugly -- %s',
$bin,
phutil_get_library_root($lib));
$symbols[$lib] = $future->resolveJSON();
}
$all_symbols = array();
foreach ($symbols as $library => $map) {
// Check for files which declare more than one class/interface in the same
// file, or mix function definitions with class/interface definitions. We
// must isolate autoloadable symbols to one per file so the autoloader
// can't end up in an unresolvable cycle.
foreach ($map as $file => $spec) {
$have = idx($spec, 'have', array());
$have_classes =
idx($have, 'class', array()) +
idx($have, 'interface', array());
$have_functions = idx($have, 'function');
if ($have_functions && $have_classes) {
$function_list = implode(', ', array_keys($have_functions));
$class_list = implode(', ', array_keys($have_classes));
$this->raiseLintInLibrary(
$library,
$file,
end($have_functions),
self::LINT_ONE_CLASS_PER_FILE,
"File '{$file}' mixes function ({$function_list}) and ".
"class/interface ({$class_list}) definitions in the same file. ".
"A file which declares a class or an interface MUST ".
"declare nothing else.");
} else if (count($have_classes) > 1) {
$class_list = implode(', ', array_keys($have_classes));
$this->raiseLintInLibrary(
$library,
$file,
end($have_classes),
self::LINT_ONE_CLASS_PER_FILE,
"File '{$file}' declares more than one class or interface ".
"({$class_list}). A file which declares a class or interface MUST ".
"declare nothing else.");
}
}
// Check for duplicate symbols: two files providing the same class or
// function.
foreach ($map as $file => $spec) {
$have = idx($spec, 'have', array());
foreach (array('class', 'function', 'interface') as $type) {
$libtype = ($type == 'interface') ? 'class' : $type;
foreach (idx($have, $type, array()) as $symbol => $offset) {
if (empty($all_symbols[$libtype][$symbol])) {
$all_symbols[$libtype][$symbol] = array(
'library' => $library,
'file' => $file,
'offset' => $offset,
);
continue;
}
$osrc = $all_symbols[$libtype][$symbol]['file'];
$olib = $all_symbols[$libtype][$symbol]['library'];
$this->raiseLintInLibrary(
$library,
$file,
$offset,
self::LINT_DUPLICATE_SYMBOL,
"Definition of {$type} '{$symbol}' in '{$file}' in library ".
"'{$library}' duplicates prior definition in '{$osrc}' in ".
"library '{$olib}'.");
}
}
}
}
foreach ($symbols as $library => $map) {
// Check for unknown symbols: uses of classes, functions or interfaces
// which are not defined anywhere. We reference the list of all symbols
// we built up earlier.
foreach ($map as $file => $spec) {
$need = idx($spec, 'need', array());
foreach (array('class', 'function', 'interface') as $type) {
$libtype = ($type == 'interface') ? 'class' : $type;
foreach (idx($need, $type, array()) as $symbol => $offset) {
if (!empty($all_symbols[$libtype][$symbol])) {
// Symbol is defined somewhere.
continue;
}
$this->raiseLintInLibrary(
$library,
$file,
$offset,
self::LINT_UNKNOWN_SYMBOL,
"Use of unknown {$type} '{$symbol}'. This symbol is not defined ".
"in any loaded phutil library. It might be misspelled, or it ".
"may have been added recently. Make sure libphutil and other ".
"libraries are up to date.");
}
}
}
}
}
private function raiseLintInLibrary($library, $path, $offset, $code, $desc) {
$root = phutil_get_library_root($library);
$this->activePath = $root.'/'.$path;
$this->raiseLintAtOffset($offset, $code, $desc);
}
public function lintPath($path) {
return;
}
public function getCacheGranularity() {
return self::GRANULARITY_GLOBAL;
}
}
diff --git a/src/workflow/ArcanistLiberateWorkflow.php b/src/workflow/ArcanistLiberateWorkflow.php
index 10d9ed32..4df0499d 100644
--- a/src/workflow/ArcanistLiberateWorkflow.php
+++ b/src/workflow/ArcanistLiberateWorkflow.php
@@ -1,230 +1,230 @@
<?php
/**
* Create and update libphutil libraries.
*
* This workflow is unusual and involves reexecuting 'arc liberate' as a
* subprocess with "--remap" and "--verify". This is because there is no way
* to unload or reload a library, so every process is stuck with the library
* definition it had when it first loaded. This is normally fine, but
* problematic in this case because 'arc liberate' modifies library definitions.
*
* @group workflow
*/
final class ArcanistLiberateWorkflow extends ArcanistBaseWorkflow {
public function getWorkflowName() {
return 'liberate';
}
public function getCommandSynopses() {
return phutil_console_format(<<<EOTEXT
**liberate** [__path__]
EOTEXT
);
}
public function getCommandHelp() {
return phutil_console_format(<<<EOTEXT
Supports: libphutil
Create or update a libphutil library, generating required metadata
files like \__init__.php.
EOTEXT
);
}
public function getArguments() {
return array(
'all' => array(
'help' =>
"Drop the module cache before liberating. This will completely ".
"reanalyze the entire library. Thorough, but slow!",
),
'force-update' => array(
'help' =>
"Force the library map to be updated, even in the presence of ".
"lint errors.",
),
'remap' => array(
'hide' => true,
'help' =>
"Internal. Run the remap step of liberation. You do not need to ".
"run this unless you are debugging the workflow.",
),
'verify' => array(
'hide' => true,
'help' =>
"Internal. Run the verify step of liberation. You do not need to ".
"run this unless you are debugging the workflow.",
),
'upgrade' => array(
'hide' => true,
'help' => "Experimental. Upgrade library to v2.",
),
'*' => 'argv',
);
}
public function run() {
$argv = $this->getArgument('argv');
if (count($argv) > 1) {
throw new ArcanistUsageException(
"Provide only one path to 'arc liberate'. The path should be a ".
"directory where you want to create or update a libphutil library.");
} else if (count($argv) == 0) {
$path = getcwd();
} else {
$path = reset($argv);
}
$is_remap = $this->getArgument('remap');
$is_verify = $this->getArgument('verify');
$path = Filesystem::resolvePath($path);
if (Filesystem::pathExists($path) && is_dir($path)) {
$init = id(new FileFinder($path))
->withPath('*/__phutil_library_init__.php')
->find();
} else {
$init = null;
}
if ($init) {
if (count($init) > 1) {
throw new ArcanistUsageException(
"Specified directory contains more than one libphutil library. Use ".
"a more specific path.");
}
$path = Filesystem::resolvePath(dirname(reset($init)), $path);
} else {
$found = false;
foreach (Filesystem::walkToRoot($path) as $dir) {
if (Filesystem::pathExists($dir.'/__phutil_library_init__.php')) {
$path = $dir;
$found = true;
break;
}
}
if (!$found) {
echo "No library currently exists at that path...\n";
$this->liberateCreateDirectory($path);
$this->liberateCreateLibrary($path);
return;
}
}
$version = $this->getLibraryFormatVersion($path);
switch ($version) {
case 1:
if ($this->getArgument('upgrade')) {
return $this->upgradeLibrary($path);
}
throw new ArcanistUsageException(
"This library is using libphutil v1, which is no longer supported. ".
"Run 'arc liberate --upgrade' to upgrade to v2.");
case 2:
if ($this->getArgument('upgrade')) {
throw new ArcanistUsageException(
"Can't upgrade a v2 library!");
}
return $this->liberateVersion2($path);
default:
throw new ArcanistUsageException(
"Unknown library version '{$version}'!");
}
}
private function getLibraryFormatVersion($path) {
$map_file = $path.'/__phutil_library_map__.php';
if (!Filesystem::pathExists($map_file)) {
// Default to library v1.
return 1;
}
$map = Filesystem::readFile($map_file);
$matches = null;
if (preg_match('/@phutil-library-version (\d+)/', $map, $matches)) {
return (int)$matches[1];
}
return 1;
}
private function liberateVersion2($path) {
$bin = $this->getScriptPath('scripts/phutil_rebuild_map.php');
return phutil_passthru(
- '%s %C %s',
+ 'php %s %C %s',
$bin,
$this->getArgument('all') ? '--drop-cache' : '',
$path);
}
private function upgradeLibrary($path) {
$inits = id(new FileFinder($path))
->withPath('*/__init__.php')
->withType('f')
->find();
echo "Removing __init__.php files...\n";
foreach ($inits as $init) {
Filesystem::remove($path.'/'.$init);
}
echo "Upgrading library to v2...\n";
$this->liberateVersion2($path);
}
private function liberateCreateDirectory($path) {
if (Filesystem::pathExists($path)) {
if (!is_dir($path)) {
throw new ArcanistUsageException(
"Provide a directory to create or update a libphutil library in.");
}
return;
}
echo "The directory '{$path}' does not exist.";
if (!phutil_console_confirm('Do you want to create it?')) {
throw new ArcanistUsageException("Cancelled.");
}
execx('mkdir -p %s', $path);
}
private function liberateCreateLibrary($path) {
$init_path = $path.'/__phutil_library_init__.php';
if (Filesystem::pathExists($init_path)) {
return;
}
echo "Creating new libphutil library in '{$path}'.\n";
echo "Choose a name for the new library.\n";
do {
$name = phutil_console_prompt('What do you want to name this library?');
if (preg_match('/^[a-z-]+$/', $name)) {
break;
} else {
echo "Library name should contain only lowercase letters and ".
"hyphens.\n";
}
} while (true);
$template =
"<?php\n\n".
"phutil_register_library('{$name}', __FILE__);\n";
echo "Writing '__phutil_library_init__.php' to '{$path}'...\n";
Filesystem::writeFile($init_path, $template);
$this->liberateVersion2($path);
}
private function getScriptPath($script) {
$root = dirname(phutil_get_library_root('arcanist'));
return $root.'/'.$script;
}
}
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sat, Oct 11, 11:13 (12 h, 22 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
984273
Default Alt Text
(27 KB)
Attached To
Mode
R118 Arcanist - fork
Attached
Detach File
Event Timeline
Log In to Comment