当使用' execute_pipeline '运行时，dagster管道将成功执行，但当使用dagi

当我通过dagit运行dagster管道时，我遇到了一个与numba编译有关的小错误，但是当我直接使用execute_pipeline时，它就不存在了。不确定如何调试它。 最小工作示例(文件dagster_umap_pipeline.py)

from dagster import solid, pipeline
from umap import UMAP
import numpy as np


@solid
def random_array(context):
    return np.random.rand(1000, 100)


@solid
def fit_umap(context, X):
    model = UMAP(15, 15)
    model.fit(X)
    return model


@pipeline
def fit_umap_pipeline():
    X = random_array()
    model = fit_umap(X)

运行从python解释器工作得很好:

>> from dagster import execute_pipeline
>> from dagster_umap_pipeline import fit_umap_pipeline
>> result = execute_pipeline(fit_umap_pipeline)
>> assert result.success  # This passes

从dagit，管道失败: py -n fit_umap_pipeline 计算失败，错误日志如下:

numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend) [1m[1mgenerator didn't yield [1m File "../miniconda3/envs/platewatch/lib/python3.6/site-packages/umap/umap_.py", line 331:[0m [1mdef compute_membership_strengths(knn_indices, knn_dists, sigmas, rhos): <source elided> rows = np.zeros((n_samples * n_neighbors), dtype=np.int64) [1m cols = np.zeros((n_samples * n_neighbors), dtype=np.int64) [0m [1m^[0m[0m [0m [0m[1m[1] During: lowering "id=1[LoopNest(index_variable = parfor_index.31, range = (0, $0.22, 1))]{281: <ir.Block at /Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/umap/umap_.py (331)>}Var(parfor_index.31, /Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/umap/umap_.py (331))" at /Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/umap/umap_.py (331)[0m ------------------------------------------------------------------------------- This should not have happened, a problem has occurred in Numba's internals. You are currently using Numba version 0.46.0. Please report the error message and traceback, along with a minimal reproducer at: https://github.com/numba/numba/issues/new If more help is needed please feel free to speak to the Numba core developers directly at: https://gitter.im/numba/numba Thanks in advance for your help in improving Numba!
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/dagster/core/errors.py", line 114, in user_code_error_boundary
    yield
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/dagster/core/engine/engine_inprocess.py", line 635, in _user_event_sequence_for_step_compute_fn
    for event in gen:
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/dagster/core/execution/plan/compute.py", line 75, in _execute_core_compute
    for step_output in _yield_compute_results(compute_context, inputs, compute_fn):
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/dagster/core/execution/plan/compute.py", line 52, in _yield_compute_results
    for event in user_event_sequence:
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/dagster/core/definitions/decorators.py", line 418, in compute
    result = fn(context, **kwargs)
  File "dagster_umap_pipeline.py", line 14, in fit_umap
    model.fit(X)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/umap/umap_.py", line 1417, in fit
    self.verbose,
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/umap/umap_.py", line 478, in fuzzy_simplicial_set
    knn_indices, knn_dists, sigmas, rhos
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/dispatcher.py", line 420, in _compile_for_args
    raise e
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/dispatcher.py", line 353, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/dispatcher.py", line 768, in compile
    cres = self._compiler.compile(args, return_type)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/dispatcher.py", line 77, in compile
    status, retval = self._compile_cached(args, return_type)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/dispatcher.py", line 91, in _compile_cached
    retval = self._compile_core(args, return_type)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/dispatcher.py", line 109, in _compile_core
    pipeline_class=self.pipeline_class)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler.py", line 528, in compile_extra
    return pipeline.compile_extra(func)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler.py", line 326, in compile_extra
    return self._compile_bytecode()
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler.py", line 385, in _compile_bytecode
    return self._compile_core()
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler.py", line 365, in _compile_core
    raise e
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler.py", line 356, in _compile_core
    pm.run(self.state)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler_machinery.py", line 328, in run
    raise patched_exception
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler_machinery.py", line 319, in run
    self._runPass(idx, pass_inst, state)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler_machinery.py", line 281, in _runPass
    mutated |= check(pss.run_pass, internal_state)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/compiler_machinery.py", line 268, in check
    mangled = func(compiler_state)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/typed_passes.py", line 380, in run_pass
    NativeLowering().run_pass(state) # TODO: Pull this out into the pipeline
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/typed_passes.py", line 325, in run_pass
    lower.lower()
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/lowering.py", line 179, in lower
    self.lower_normal_function(self.fndesc)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/lowering.py", line 220, in lower_normal_function
    entry_block_tail = self.lower_function_body()
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/lowering.py", line 245, in lower_function_body
    self.lower_block(block)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/lowering.py", line 260, in lower_block
    self.lower_inst(inst)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/errors.py", line 725, in new_error_context
    six.reraise(type(newerr), newerr, tb)
  File "/Users/ben.fogelson/miniconda3/envs/platewatch/lib/python3.6/site-packages/numba/six.py", line 669, in reraise
    raise value

相关包版本:

umap-learn                0.3.10                   py36_0    conda-forge
numba                     0.46.0           py36h6440ff4_0
dagster                   0.6.6                    py36_0    conda-forge
dagit                     0.6.6                      py_0    conda-forge

问题来源StackOverflow 地址：/questions/59379523/dagster-pipeline-executes-successfully-when-run-with-execute-pipeline-but-not