diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81431686..cf47cdac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,6 @@ stages: + # Temporary, just so we fail fast + - complement - ci - artifacts - publish @@ -134,6 +136,29 @@ artifacts: - if: $CI interruptible: true +complement: + stage: complement + image: docker:28.3.2 + services: + - docker:28.3.2-dind + script: + - apk add coreutils bash jq go + - git clone https://github.com/matrix-org/complement complement-clone + # TODO: Better way of pinning commit, ideally to make it easily accessible in a local dev env + - env -C complement-clone git checkout bdeba288ae43955f7e707121ef004b90482a8f37 + - bin/complement complement-clone complement-log.jsonl complement-results.jsonl + # TODO: Nicer output of some sort, i.e. moving this entire stage to a script (also could solve the problem above?) + - diff tests/complement/results.jsonl complement-results.jsonl + rules: + # CI required for all MRs + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Optional CI on forks + - if: $IS_UPSTREAM_CI != "true" + when: manual + allow_failure: true + - if: $CI + interruptible: true + .push-oci-image: stage: publish image: docker:25.0.0 diff --git a/tests/complement/results.jsonl b/tests/complement/results.jsonl new file mode 100644 index 00000000..72b9eeae --- /dev/null +++ b/tests/complement/results.jsonl @@ -0,0 +1,90 @@ +{"Test":"TestACLs","Action":"pass"} +{"Test":"TestBannedUserCannotSendJoin","Action":"fail"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1/event_with_mismatched_state_key","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1/invite_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1/knock_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1/leave_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1/non-state_membership_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV1/regular_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2/event_with_mismatched_state_key","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2/invite_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2/knock_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2/leave_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2/non-state_membership_event","Action":"pass"} +{"Test":"TestCannotSendNonJoinViaSendJoinV2/regular_event","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/event_with_mismatched_state_key","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/invite_event","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/join_event","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/knock_event","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/non-state_membership_event","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/regular_event","Action":"fail"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/event_with_mismatched_state_key","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/invite_event","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/join_event","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event","Action":"pass"} +{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event","Action":"pass"} +{"Test":"TestContentMediaV1","Action":"pass"} +{"Test":"TestDeviceListsUpdateOverFederation","Action":"fail"} +{"Test":"TestDeviceListsUpdateOverFederation/good_connectivity","Action":"fail"} +{"Test":"TestDeviceListsUpdateOverFederation/interrupted_connectivity","Action":"fail"} +{"Test":"TestDeviceListsUpdateOverFederation/stopped_server","Action":"fail"} +{"Test":"TestDeviceListsUpdateOverFederationOnRoomJoin","Action":"fail"} +{"Test":"TestEventAuth","Action":"pass"} +{"Test":"TestEventAuth/returns_auth_events_for_the_requested_event","Action":"pass"} +{"Test":"TestEventAuth/returns_the_auth_chain_for_the_requested_event","Action":"pass"} +{"Test":"TestFederationRedactSendsWithoutEvent","Action":"pass"} +{"Test":"TestFederationRejectInvite","Action":"pass"} +{"Test":"TestFederationRoomsInvite","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_join_the_room_when_homeserver_is_already_participating_in_the_room","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_reject_invite_when_homeserver_is_already_participating_in_the_room","Action":"pass"} +{"Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata","Action":"pass"} +{"Test":"TestGetMissingEventsGapFilling","Action":"fail"} +{"Test":"TestInboundCanReturnMissingEvents","Action":"fail"} +{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility","Action":"fail"} +{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_joined_visibility","Action":"fail"} +{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_shared_visibility","Action":"fail"} +{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_world_readable_visibility","Action":"fail"} +{"Test":"TestInboundFederationKeys","Action":"pass"} +{"Test":"TestInboundFederationProfile","Action":"pass"} +{"Test":"TestInboundFederationProfile/Inbound_federation_can_query_profile_data","Action":"pass"} +{"Test":"TestInboundFederationProfile/Non-numeric_ports_in_server_names_are_rejected","Action":"pass"} +{"Test":"TestInboundFederationRejectsEventsWithRejectedAuthEvents","Action":"fail"} +{"Test":"TestIsDirectFlagFederation","Action":"pass"} +{"Test":"TestIsDirectFlagLocal","Action":"pass"} +{"Test":"TestJoinFederatedRoomFailOver","Action":"pass"} +{"Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser","Action":"fail"} +{"Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user","Action":"fail"} +{"Test":"TestJoinFederatedRoomWithUnverifiableEvents","Action":"pass"} +{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join","Action":"pass"} +{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join","Action":"pass"} +{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join","Action":"pass"} +{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join","Action":"pass"} +{"Test":"TestJoinViaRoomIDAndServerName","Action":"pass"} +{"Test":"TestNetworkPartitionOrdering","Action":"fail"} +{"Test":"TestOutboundFederationEventSizeGetMissingEvents","Action":"fail"} +{"Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6","Action":"fail"} +{"Test":"TestOutboundFederationProfile","Action":"pass"} +{"Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data","Action":"pass"} +{"Test":"TestOutboundFederationSend","Action":"pass"} +{"Test":"TestRemoteAliasRequestsUnderstandUnicode","Action":"pass"} +{"Test":"TestRemotePresence","Action":"fail"} +{"Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members","Action":"fail"} +{"Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members","Action":"fail"} +{"Test":"TestRemoteTyping","Action":"fail"} +{"Test":"TestSendJoinPartialStateResponse","Action":"skip"} +{"Test":"TestSyncOmitsStateChangeOnFilteredEvents","Action":"pass"} +{"Test":"TestToDeviceMessagesOverFederation/good_connectivity","Action":"pass"} +{"Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity","Action":"fail"} +{"Test":"TestUnbanViaInvite","Action":"pass"} +{"Test":"TestUserAppearsInChangedDeviceListOnJoinOverFederation","Action":"pass"} +{"Test":"TestWriteMDirectAccountData","Action":"pass"}