WIP: ci: run complement

.gitlab-ci.yml

@@ -1,4 +1,6 @@
 stages:
+  # Temporary, just so we fail fast
+  - complement
   - ci
   - artifacts
   - publish
@@ -134,6 +136,29 @@ artifacts:
     - if: $CI
   interruptible: true
 
+complement:
+  stage: complement
+  image: docker:28.3.2
+  services:
+    - docker:28.3.2-dind
+  script:
+    - apk add coreutils bash jq go
+    - git clone https://github.com/matrix-org/complement complement-clone
+    # TODO: Better way of pinning commit, ideally to make it easily accessible in a local dev env
+    - env -C complement-clone git checkout bdeba288ae43955f7e707121ef004b90482a8f37
+    - bin/complement complement-clone complement-log.jsonl complement-results.jsonl
+    # TODO: Nicer output of some sort, i.e. moving this entire stage to a script (also could solve the problem above?)
+    - diff tests/complement/results.jsonl complement-results.jsonl
+  rules:
+    # CI required for all MRs
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    # Optional CI on forks
+    - if: $IS_UPSTREAM_CI != "true"
+      when: manual
+      allow_failure: true
+    - if: $CI
+  interruptible: true
+
 .push-oci-image:
   stage: publish
   image: docker:25.0.0

tests/complement/results.jsonl

@@ -0,0 +1,90 @@
+{"Test":"TestACLs","Action":"pass"}
+{"Test":"TestBannedUserCannotSendJoin","Action":"fail"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1/event_with_mismatched_state_key","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1/invite_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1/knock_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1/leave_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1/non-state_membership_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV1/regular_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2/event_with_mismatched_state_key","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2/invite_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2/knock_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2/leave_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2/non-state_membership_event","Action":"pass"}
+{"Test":"TestCannotSendNonJoinViaSendJoinV2/regular_event","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/event_with_mismatched_state_key","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/invite_event","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/join_event","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/knock_event","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/non-state_membership_event","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV1/regular_event","Action":"fail"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/event_with_mismatched_state_key","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/invite_event","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/join_event","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event","Action":"pass"}
+{"Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event","Action":"pass"}
+{"Test":"TestContentMediaV1","Action":"pass"}
+{"Test":"TestDeviceListsUpdateOverFederation","Action":"fail"}
+{"Test":"TestDeviceListsUpdateOverFederation/good_connectivity","Action":"fail"}
+{"Test":"TestDeviceListsUpdateOverFederation/interrupted_connectivity","Action":"fail"}
+{"Test":"TestDeviceListsUpdateOverFederation/stopped_server","Action":"fail"}
+{"Test":"TestDeviceListsUpdateOverFederationOnRoomJoin","Action":"fail"}
+{"Test":"TestEventAuth","Action":"pass"}
+{"Test":"TestEventAuth/returns_auth_events_for_the_requested_event","Action":"pass"}
+{"Test":"TestEventAuth/returns_the_auth_chain_for_the_requested_event","Action":"pass"}
+{"Test":"TestFederationRedactSendsWithoutEvent","Action":"pass"}
+{"Test":"TestFederationRejectInvite","Action":"pass"}
+{"Test":"TestFederationRoomsInvite","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_join_the_room_when_homeserver_is_already_participating_in_the_room","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_reject_invite_when_homeserver_is_already_participating_in_the_room","Action":"pass"}
+{"Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata","Action":"pass"}
+{"Test":"TestGetMissingEventsGapFilling","Action":"fail"}
+{"Test":"TestInboundCanReturnMissingEvents","Action":"fail"}
+{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility","Action":"fail"}
+{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_joined_visibility","Action":"fail"}
+{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_shared_visibility","Action":"fail"}
+{"Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_world_readable_visibility","Action":"fail"}
+{"Test":"TestInboundFederationKeys","Action":"pass"}
+{"Test":"TestInboundFederationProfile","Action":"pass"}
+{"Test":"TestInboundFederationProfile/Inbound_federation_can_query_profile_data","Action":"pass"}
+{"Test":"TestInboundFederationProfile/Non-numeric_ports_in_server_names_are_rejected","Action":"pass"}
+{"Test":"TestInboundFederationRejectsEventsWithRejectedAuthEvents","Action":"fail"}
+{"Test":"TestIsDirectFlagFederation","Action":"pass"}
+{"Test":"TestIsDirectFlagLocal","Action":"pass"}
+{"Test":"TestJoinFederatedRoomFailOver","Action":"pass"}
+{"Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser","Action":"fail"}
+{"Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user","Action":"fail"}
+{"Test":"TestJoinFederatedRoomWithUnverifiableEvents","Action":"pass"}
+{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join","Action":"pass"}
+{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join","Action":"pass"}
+{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join","Action":"pass"}
+{"Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join","Action":"pass"}
+{"Test":"TestJoinViaRoomIDAndServerName","Action":"pass"}
+{"Test":"TestNetworkPartitionOrdering","Action":"fail"}
+{"Test":"TestOutboundFederationEventSizeGetMissingEvents","Action":"fail"}
+{"Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6","Action":"fail"}
+{"Test":"TestOutboundFederationProfile","Action":"pass"}
+{"Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data","Action":"pass"}
+{"Test":"TestOutboundFederationSend","Action":"pass"}
+{"Test":"TestRemoteAliasRequestsUnderstandUnicode","Action":"pass"}
+{"Test":"TestRemotePresence","Action":"fail"}
+{"Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members","Action":"fail"}
+{"Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members","Action":"fail"}
+{"Test":"TestRemoteTyping","Action":"fail"}
+{"Test":"TestSendJoinPartialStateResponse","Action":"skip"}
+{"Test":"TestSyncOmitsStateChangeOnFilteredEvents","Action":"pass"}
+{"Test":"TestToDeviceMessagesOverFederation/good_connectivity","Action":"pass"}
+{"Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity","Action":"fail"}
+{"Test":"TestUnbanViaInvite","Action":"pass"}
+{"Test":"TestUserAppearsInChangedDeviceListOnJoinOverFederation","Action":"pass"}
+{"Test":"TestWriteMDirectAccountData","Action":"pass"}