no ideas?

On Wed, Aug 28, 2019 at 10:45 PM Nick Chase <nchase@mirantis.com> wrote:
OK, unfortunately I'm wrong, it does NOT work. :(  Horizon does come up, and the sample instance is there, but the actual Airship URLs are still giving me the "default backend - 404" error.   I found the following in the airflow-web container in the shipyard-api pod, which is in a crashloop:

2019-08-29 02:42:23,471 ERROR    models.py:380:process_file models Failed to import: /usr/local/airflow/dags/redeploy_server.py

Traceback (most recent call last):

  File "/usr/local/lib/python3.5/dist-packages/airflow/models.py", line 377, in process_file

    m = imp.load_source(mod_name, filepath)

  File "/usr/lib/python3.5/imp.py", line 172, in load_source

    module = _load(spec)

  File "<frozen importlib._bootstrap>", line 693, in _load

  File "<frozen importlib._bootstrap>", line 673, in _load_unlocked

  File "<frozen importlib._bootstrap_external>", line 665, in exec_module

  File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed

  File "/usr/local/airflow/dags/redeploy_server.py", line 57, in <module>

    targets=[BAREMETAL]

  File "/usr/local/airflow/dags/common_step_factory.py", line 171, in get_validate_site_design

    dag=self.dag)

  File "/usr/local/lib/python3.5/dist-packages/airflow/utils/db.py", line 74, in wrapper

    return func(*args, **kwargs)

  File "/usr/lib/python3.5/contextlib.py", line 66, in __exit__

    next(self.gen)

  File "/usr/local/lib/python3.5/dist-packages/airflow/utils/db.py", line 45, in create_session

    session.commit()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 943, in commit

    self.transaction.commit()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 465, in commit

    self._assert_active(prepared_ok=True)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 276, in _assert_active

    % self._rollback_exception

sqlalchemy.exc.InvalidRequestError: This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg2.OperationalError) could not connect to server: Connection refused

Is the server running on host "postgresql.ucp.svc.cluster.local" (10.96.212.170) and accepting

TCP/IP connections on port 5432?

 (Background on this error at: http://sqlalche.me/e/e3q8)

2019-08-29 02:42:23,507 ERROR    models.py:380:process_file models Failed to import: /usr/local/airflow/dags/samples/airflow_task_state_operators.py

Traceback (most recent call last):

  File "/usr/local/lib/python3.5/dist-packages/airflow/models.py", line 377, in process_file

    m = imp.load_source(mod_name, filepath)

  File "/usr/lib/python3.5/imp.py", line 172, in load_source

    module = _load(spec)

  File "<frozen importlib._bootstrap>", line 693, in _load

  File "<frozen importlib._bootstrap>", line 673, in _load_unlocked

  File "<frozen importlib._bootstrap_external>", line 665, in exec_module

  File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed

  File "/usr/local/airflow/dags/samples/airflow_task_state_operators.py", line 19, in <module>

    from airflow.operators import TaskStateOperator

ImportError: cannot import name 'TaskStateOperator'

/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py:144: UserWarning: The psycopg2 wheel package will be renamed from release 2.8; in order to keep installing from binary please use "pip install psycopg2-binary" instead. For details see: <http://initd.org/psycopg/docs/install.html#binary-install-from-pypi>.

  """)

Traceback (most recent call last):

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect

    return fn()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 400, in connect

    return _ConnectionFairy._checkout(self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 788, in _checkout

    fairy = _ConnectionRecord.checkout(pool)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 529, in checkout

    rec = pool._do_get()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get

    self._dec_overflow()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__

    compat.reraise(exc_type, exc_value, exc_tb)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 249, in reraise

    raise value

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 1190, in _do_get

    return self._create_connection()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 347, in _create_connection

    return _ConnectionRecord(self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 474, in __init__

    self.__connect(first_connect_check=True)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 671, in __connect

    connection = pool._invoke_creator(self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect

    return dialect.connect(*cargs, **cparams)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 412, in connect

    return self.dbapi.connect(*cargs, **cparams)

  File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect

    conn = _connect(dsn, connection_factory=connection_factory, **kwasync)

psycopg2.OperationalError: could not connect to server: Connection refused

Is the server running on host "postgresql.ucp.svc.cluster.local" (10.96.212.170) and accepting

TCP/IP connections on port 5432?



The above exception was the direct cause of the following exception:


Traceback (most recent call last):

  File "/usr/local/bin/airflow", line 32, in <module>

    args.func(args)

  File "/usr/local/lib/python3.5/dist-packages/airflow/utils/cli.py", line 74, in wrapper

    return f(*args, **kwargs)

  File "/usr/local/lib/python3.5/dist-packages/airflow/bin/cli.py", line 793, in webserver

    app = cached_app_rbac(conf) if settings.RBAC else cached_app(conf)

  File "/usr/local/lib/python3.5/dist-packages/airflow/www/app.py", line 187, in cached_app

    app = create_app(config, testing)

  File "/usr/local/lib/python3.5/dist-packages/airflow/www/app.py", line 72, in create_app

    from airflow.www import views

  File "/usr/local/lib/python3.5/dist-packages/airflow/www/views.py", line 2232, in <module>

    class ChartModelView(wwwutils.DataProfilingMixin, AirflowModelView):

  File "/usr/local/lib/python3.5/dist-packages/airflow/www/views.py", line 2323, in ChartModelView

    'conn_id': _connection_ids()

  File "/usr/local/lib/python3.5/dist-packages/airflow/utils/db.py", line 74, in wrapper

    return func(*args, **kwargs)

  File "/usr/local/lib/python3.5/dist-packages/airflow/www/views.py", line 2227, in _connection_ids

    .group_by(models.Connection.conn_id)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/query.py", line 2995, in __iter__

    return self._execute_and_instances(context)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/query.py", line 3016, in _execute_and_instances

    close_with_result=True)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/query.py", line 3025, in _get_bind_args

    **kw

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/query.py", line 3007, in _connection_from_session

    conn = self.session.connection(**kw)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 1035, in connection

    execution_options=execution_options)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 1040, in _connection_for_bind

    engine, execution_options)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/orm/session.py", line 409, in _connection_for_bind

    conn = bind.contextual_connect()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 2123, in contextual_connect

    self._wrap_pool_connect(self.pool.connect, None),

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 2162, in _wrap_pool_connect

    e, dialect, self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1476, in _handle_dbapi_exception_noconnection

    exc_info

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 265, in raise_from_cause

    reraise(type(exception), exception, tb=exc_tb, cause=cause)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 248, in reraise

    raise value.with_traceback(tb)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect

    return fn()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 400, in connect

    return _ConnectionFairy._checkout(self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 788, in _checkout

    fairy = _ConnectionRecord.checkout(pool)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 529, in checkout

    rec = pool._do_get()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 1193, in _do_get

    self._dec_overflow()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__

    compat.reraise(exc_type, exc_value, exc_tb)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 249, in reraise

    raise value

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 1190, in _do_get

    return self._create_connection()

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 347, in _create_connection

    return _ConnectionRecord(self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 474, in __init__

    self.__connect(first_connect_check=True)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/pool.py", line 671, in __connect

    connection = pool._invoke_creator(self)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/strategies.py", line 106, in connect

    return dialect.connect(*cargs, **cparams)

  File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 412, in connect

    return self.dbapi.connect(*cargs, **cparams)

  File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect

    conn = _connect(dsn, connection_factory=connection_factory, **kwasync)

sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) could not connect to server: Connection refused

Is the server running on host "postgresql.ucp.svc.cluster.local" (10.96.212.170) and accepting

TCP/IP connections on port 5432?

 (Background on this error at: http://sqlalche.me/e/e3q8)



The postgresql pod appears to be running.


Any thoughts?


----  Nick

<snip>