Compare commits

..
349 Commits
Author SHA1 Message Date
NotAlexNoyle c645ed1adf Exclude CODEOWNERS from language statistics 2026-11-01 09:18:32 -05:00
NotAlexNoyle 87234d9f76 Add CODEOWNERS (one staff approval) 2026-11-01 09:18:32 -05:00
Alexey Palazhchenko 0d5eb3343a Prepare v2.7.0 release (#5538) 2025-11-10 21:18:24 +04:00
Alexey Palazhchenko 52ad11f0af Bump deps (#5537) 2025-11-10 20:45:21 +04:00
Alexey Palazhchenko 71b6eb3a08 Bump DocumentDB (#5536) 2025-11-09 22:46:54 +04:00
5cb62a1155 Add MCP tools (#5452)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-11-09 20:36:10 +04:00
Alexey Palazhchenko 867537dd01 Use Go 1.25.4 (#5534) 2025-11-08 09:35:59 +04:00
487880fd19 Add basic MCP server (#5279)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-11-06 09:08:44 +04:00
Chi FujiiandAlexey Palazhchenko 3b29cece0c Refactor DataAPI with graceful shutdown and add OpenAPI specification endpoint (#5357)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-10-27 18:55:12 +04:00
Alexey Palazhchenko c1bb0b6db6 Update DocumentDB (#5528) 2025-10-22 18:46:58 +04:00
Alexey Palazhchenko a04b20a58f Use forked lazyerrors (#5527) 2025-10-22 16:13:02 +04:00
Josh Soref 2b2ce550d4 Fix CODEOWNERS grammar (#5495)
Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
2025-10-21 12:51:38 +04:00
Alexey Palazhchenko 87293d3434 Skip flaky test for now (#5526) 2025-10-21 12:50:19 +04:00
Alexey Palazhchenko 7b0d200262 Use Go 1.25.3 (#5525) 2025-10-21 11:43:35 +04:00
Patryk KwiatekandAlexey Palazhchenko d1d037a030 Add a test for *documentdb.Conn finalizing (#5484)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-10-21 11:28:03 +04:00
Alexey Palazhchenko c37a62b0a2 Bump dependencies (#5519) 2025-10-09 17:57:31 +04:00
Alexey Palazhchenko cff1577d19 Improve proxy mode observability (#5440) 2025-10-03 19:53:25 +04:00
Alexey Palazhchenko efa50d4877 Add tracing of SQL queries (#5514) 2025-10-02 21:02:46 +04:00
Alexey Palazhchenko 067eb2f143 Rename OpenTelemetry spans (#5512) 2025-09-29 14:58:13 +04:00
Alexey Palazhchenko 4e8e51751d Minor observability cleanup (#5511) 2025-09-28 22:32:36 +03:00
Alexey Palazhchenko ae0b94de12 Bump deps (#5509) 2025-09-28 20:56:00 +03:00
Alexey Palazhchenko 2c24483c06 Use Go 1.25.1 (#5508) 2025-09-28 15:48:00 +03:00
Josh Soref 221d5cbd08 Fix spelling in .go files (#5492)
Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
2025-09-08 10:33:12 +09:00
Alexey Palazhchenko 62ce1019b1 Simplify licensing 2025-09-07 12:08:29 +04:00
Josh Soref ef9934e3ed Replace malicious container registry references with ghcr (#5486) 2025-09-03 20:05:21 +04:00
Alexey Palazhchenko e78eaffb58 Finalize PostgreSQL metrics (#5485) 2025-09-03 15:25:44 +04:00
Chi FujiiandAlexey Palazhchenko 370b1586c5 Add PostgreSQL metrics (#5427)
Closes #5402.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-09-02 14:50:16 +04:00
Alexander Tobi FashakinandCopilot 1f8d160296 Update cloud installation docs (#5466)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-09-01 19:52:16 +04:00
3656ebd1c4 Use generated structs in Data API (#5382)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-09-01 13:24:21 +04:00
Patryk KwiatekandAlexey Palazhchenko e3a7406b9e Update wire dependency (#5465)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-09-01 12:48:10 +04:00
Alexey Palazhchenko 4847e59e25 Update dependencies (#5479) 2025-09-01 11:55:26 +04:00
Alexey Palazhchenko b6fa806add Move to documentdb/documentdb (#5468) 2025-08-27 12:35:57 +04:00
Chi FujiiandAlexey Palazhchenko 5394f00550 Avoid hijacking connections from the pool and immediately closing them (#5443)
Closes #5402.
Closes FerretDB/FerretDB-DocumentDB#270.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-08-27 11:37:26 +04:00
Alexey Palazhchenko 9a48b7132c Add helpers for lazy logging (#5467) 2025-08-27 10:26:35 +04:00
Alexey Palazhchenko 4cea93524d Fix FerretDB Cloud link 🤦 2025-08-26 19:38:39 +04:00
Alexey Palazhchenko 3cd4554d87 Add Cease and Desist Notice (#5462) 2025-08-25 18:50:08 +04:00
Alexey Palazhchenko 2e6ca25078 Add FerretDB Cloud blog post (#5461) 2025-08-25 17:04:31 +04:00
Alexey Palazhchenko 901a8f6030 Add MongoDB's complaint (#5460) 2025-08-25 12:45:03 +04:00
Alexey Palazhchenko b1feb6377d Add documentation page stub for MCP server (#5451) 2025-08-22 07:49:36 +00:00
Alexey Palazhchenko 3fd8c982a5 Bump deps (#5455) 2025-08-22 11:32:52 +04:00
Alexander Tobi FashakinandCopilot c05de8bdea Document cloud instance deployment and setup (#5424)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-08-21 20:15:06 +04:00
Alexey Palazhchenko d97f2612b8 Add TODO comment about Debian 13 Trixie (#5450)
Refs FerretDB/FerretDB#5449.
2025-08-21 13:42:59 +04:00
Alexey Palazhchenko 6d8e86b774 Remove flaky test (#5444) 2025-08-20 12:17:35 +04:00
Alexander Tobi Fashakin 7a117e768d Update compatible apps (#5442) 2025-08-20 05:51:21 +00:00
Alexander Tobi FashakinandCopilot bfd1650ee2 Update content guide (#5441)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-08-19 19:09:52 +04:00
Alexey Palazhchenko d9a14c883e Add comments to supervisord config (#5430) 2025-08-19 14:07:17 +04:00
Alexey Palazhchenko 2b652dae20 Use middleware (#5377) 2025-08-19 13:57:06 +04:00
Alexey Palazhchenko 6631594402 Add middleware (#5437) 2025-08-19 07:38:31 +04:00
c5f69fe962 Add blog post on Formio (#5395)
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-08-18 22:39:46 +04:00
Alexey Palazhchenko a65042d381 Update expected PostgreSQL version (#5438) 2025-08-18 20:12:33 +04:00
Alexey Palazhchenko 24981ad991 Tweak conninfo (#5435) 2025-08-18 12:23:23 +04:00
Alexey Palazhchenko 2078f859f3 Make Run wait for all Handle calls to finish (#5434) 2025-08-18 11:52:42 +04:00
Chi FujiiandAlexey Palazhchenko 16670ff34f Upload YugabyteDB log files (#5401)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-08-18 09:27:56 +04:00
Alexey Palazhchenko b994d18792 Update DocumentDB (#5431) 2025-08-14 19:21:04 +04:00
Alexey Palazhchenko 4ea90be4aa Use QEMU on arm64 for Yugabyte for now (#5429) 2025-08-14 12:32:49 +04:00
3770c1e4eb Add release blog post for FerretDB v2.5.0 (#5412)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-08-14 12:06:53 +04:00
Chi Fujii 5fd75c160a Bump YugabyteDB to stable version (#5428) 2025-08-14 10:55:16 +04:00
Alexey Palazhchenko c60131a69c Update documentation to point to the next release (#5423) 2025-08-14 10:54:45 +04:00
Alexey Palazhchenko b4eda04f37 Prepare v2.5.0 release (#5414) 2025-08-12 15:06:25 +04:00
Alexey Palazhchenko 6e2d3affd9 Update DocumentDB (#5411) 2025-08-12 10:25:34 +04:00
Alexey Palazhchenko 2cbf8cac53 Update Prometheus metrics (#5403) 2025-08-11 19:51:18 +04:00
Patryk Kwiatek b933b1f365 Test flaky tests detection (#5393) 2025-08-06 14:12:10 +04:00
Alexey Palazhchenko 96d34a4615 Enable MongoDB test commands (#5400) 2025-08-04 11:17:26 +04:00
Alexey Palazhchenko 1a5a6ec3c3 Update DocumentDB (#5398) 2025-08-01 18:29:12 +04:00
Chi Fujii 997c02f379 Run all YugabyteDB integration tests (#5283) 2025-08-01 14:31:33 +04:00
Alexey Palazhchenko 2412bf3595 Make a resource.Untrack thread-safe (#5396) 2025-08-01 12:10:01 +04:00
Alexey Palazhchenko 20697b6b74 Small tweaks (#5394) 2025-07-31 09:53:28 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko cb18c69dcc Add WeKan blog post (#5381)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-07-30 19:32:21 +04:00
Alexey Palazhchenko e99a07db05 Bump DocumentDB version (#5392) 2025-07-30 19:13:25 +04:00
Chi FujiiandAlexey Palazhchenko b1189f2594 Add YugabyteDB integration tests (#5274)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-07-30 16:53:31 +04:00
Alexey Palazhchenko 783d39086c Encapsulate PostgreSQL pool in the Handler (#5389) 2025-07-30 11:28:47 +04:00
Alexey Palazhchenko c87f9e3c18 Small cleanup (#5388) 2025-07-30 11:02:43 +04:00
Patryk Kwiatek e5615b35ca Use Codecov for flaky integration tests detection (#5361) 2025-07-30 05:51:39 +00:00
Alexey PalazhchenkoandCopilot 327a8fb759 Extract wiring into a separate package (#5383)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-30 09:23:06 +04:00
Alexander Tobi Fashakin 319128c43f Enable full visibility for images on zoom (#5346) 2025-07-28 20:21:27 +04:00
Alexey Palazhchenko bccd6276dd Update DocumentDB (#5380) 2025-07-28 12:17:48 +04:00
Alexey Palazhchenko 1b5913f2bf Simplify flags handling (#5378) 2025-07-28 09:16:48 +04:00
Chi Fujii 66f9bc149d Make accessing YugabyteDB logs easier (#5362)
Closes #85.
2025-07-25 08:12:33 +00:00
Chi Fujii f06bf851a9 Skip YugabyteDB setup for arm64 (#5372)
Closes #85.
2025-07-25 07:06:06 +00:00
Alexey Palazhchenko c891abbdf6 Refactor normal and error responses (#5326) 2025-07-24 19:24:29 +04:00
Alexey PalazhchenkoandChi Fujii f65c7b643a Tweak Ollama and MCPHost setup (#5375)
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
2025-07-24 09:06:14 +04:00
Alexander Tobi Fashakin bc0e9128e3 Add Vault blog post (#5368)
Closes #5152.
2025-07-23 19:44:59 +04:00
Patryk Kwiatek e1ea96acf4 Add components in codecov (#5364) 2025-07-23 11:06:56 +04:00
Alexey Palazhchenko f64490cf94 Update DocumentDB (#5353) 2025-07-17 12:14:08 +04:00
Alexander Tobi Fashakin 5a05644fed Add release blogpost for 2.4.0 (#5354) 2025-07-16 19:56:04 +04:00
Alexey Palazhchenko 9a45f6bc26 Prepare v2.4.0 release (#5344) 2025-07-16 11:31:22 +04:00
d9d3a9f8dd Add HyperDX blog post (#5334)
Closes #5226.

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-07-15 20:19:07 +04:00
Alexey Palazhchenko 91feb6e54a Update DocumentDB (#5342) 2025-07-15 17:13:59 +04:00
Alexander Tobi Fashakin fc1ec1ad61 Add compatible applications (#5335) 2025-07-10 08:06:09 +00:00
Chi Fujii b1afd1f15a Add YugabyteDB to local setup (#5258)
Closes #85.
2025-07-09 09:44:12 +00:00
Alexey PalazhchenkoandCopilot 746c84a0d6 Split command handling and handlers (#5328)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-09 11:05:44 +04:00
2936407d35 Add DBeaver compatibility blogpost (#5321)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-07-08 20:06:47 +04:00
Alexey Palazhchenko 07449f2391 Avoid extra marshaling (#5327) 2025-07-08 19:28:47 +04:00
Chi FujiiandAlexey Palazhchenko c62a0ec6ba Allow users to perform user management commands (#5314)
Closes #85.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-07-08 09:11:38 +04:00
Alexander Tobi Fashakin 9ae0a666e9 Add GrowthBook to compatible apps (#5319) 2025-07-04 08:10:04 +00:00
Chi Fujii 13b36b44ef Update TODO URL for dropUsers test (#5324) 2025-07-04 11:59:07 +04:00
Alexey Palazhchenko 1916b89173 Remove must.NotFail(wire[bson].NewXXX( pattern (#5318) 2025-07-04 08:49:53 +09:00
d558f38ee8 Add Heyform compatibility blog post (#5316)
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-07-03 18:39:38 +04:00
Alexey Palazhchenko 7eacf8cefc Refactor responses handling (#5310) 2025-07-03 14:57:18 +04:00
Chi Fujii d6245127ac Update TODO URLs (#5317) 2025-07-02 09:43:56 +04:00
Alexey Palazhchenko f23e436f39 Do not use deprecated method (#5312) 2025-07-01 19:57:09 +04:00
Alexey Palazhchenko a630f68a9f Refactor requests handling (#5292) 2025-06-30 13:22:54 +04:00
Alexey Palazhchenko 7f7072c83a Bump deps (#5307) 2025-06-29 16:25:31 +02:00
Alexander Tobi Fashakin f248bb1356 Add Growthbook blogpost (#5304) 2025-06-27 17:57:33 +02:00
Alexey Palazhchenko b4bd17b678 Minor refactoring of Data API (#5302) 2025-06-27 11:23:57 +02:00
Alexey PalazhchenkoandCopilot 2f974ee36e Extend middleware requests and responses (#5301)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-06-27 08:47:14 +02:00
Chi Fujii cec3175583 Cleanup NodeBB blog post (#5294) 2025-06-26 10:53:19 +00:00
Alexey Palazhchenko 969cf59ef3 Add Dependabot configuration for mcphost (#5296) 2025-06-26 11:02:38 +02:00
Chi Fujii 169e5d3800 Bring back Codecov and Coveralls integration (#5276) 2025-06-26 09:57:18 +02:00
Chi Fujii 1618cf7b19 Add mcphost as tool (#5287) 2025-06-26 09:56:31 +02:00
Alexander Tobi Fashakin 0d84f2c324 Add NodeBB to compatible apps (#5293) 2025-06-25 16:52:11 +02:00
Alexander Tobi Fashakin 871f60d169 Add NodeBB compatibility blog post (#5285) 2025-06-25 14:39:00 +02:00
Alexander Tobi Fashakin e66693f38b Add Librechat to compatibility section (#5286) 2025-06-25 09:30:36 +02:00
Alexander Tobi Fashakin 30bdf62714 Add blog post on setting up LibreChat with FerretDB (#4999) 2025-06-23 16:42:37 +02:00
Alexander Tobi Fashakin c4b9de7f45 Add PayloadCMS to compatible apps (#5270) 2025-06-23 12:22:54 +00:00
Alexey Palazhchenko 211b1348a5 Update DocumentDB (#5282) 2025-06-20 16:38:45 +04:00
Alexey Palazhchenko eb59f61d73 Wrap long lines in a blog post (#5280) 2025-06-20 13:44:29 +04:00
Alexander Tobi Fashakin d539b13641 Add Compass to compatible apps (#5275) 2025-06-19 12:29:17 +04:00
Alexander Tobi Fashakin 117d335fa3 Add blog post on using Compass GUI (#5263) 2025-06-18 19:43:51 +04:00
Alexey Palazhchenko 0dc0844eb3 Use debug-level log messages for continuations (#5266)
Closes #5265.
2025-06-17 13:35:06 +00:00
Alexey Palazhchenko c58bcdb8e8 Remove skipping of bypassEmptyTsReplacement (#5267) 2025-06-17 11:19:38 +00:00
Alexander Tobi Fashakin c765a2fb06 Add blog post for Payload CMS (#5257) 2025-06-17 10:20:03 +04:00
Alexey Palazhchenko 89eb76043c Update DocumentDB (#5268) 2025-06-16 17:35:10 +04:00
Alexander Tobi Fashakin 84f3b9cf7c Fix social links (#5264) 2025-06-16 07:05:48 +00:00
Alexey Palazhchenko 51335dc57f Update DocumentDB (#5261) 2025-06-16 10:21:56 +04:00
Alexey Palazhchenko 0ad2ad90fa Revert "Remove hack for bypassEmptyTsReplacement (#5217)" (#5256) 2025-06-12 17:03:53 +04:00
Alexey Palazhchenko a3132614c4 Prepare v2.3.0 release (#5254) 2025-06-11 15:33:19 +04:00
Alexey Palazhchenko 761df3603b Improve telemetry settings for embedded package (#5252) 2025-06-11 14:52:45 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko 6adbbf8847 Create compatible apps section (#5108)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-06-11 14:39:27 +04:00
Alexey Palazhchenko e089a11124 Remove hack for bypassEmptyTsReplacement (#5217) 2025-06-11 13:35:30 +04:00
Alexey Palazhchenko 14f728e94c Build production Docker images for PRs (#5251) 2025-06-11 12:57:58 +04:00
Alexey Palazhchenko 7a8bb089ad Use DocumentDB's compat command (#5242) 2025-06-11 12:38:00 +04:00
Alexey Palazhchenko b16ee73b52 Update DocumentDB (#5240) 2025-06-10 11:40:22 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko 416c6384fa Add blog post for Novu compatibility (#5195)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-06-06 16:50:20 +04:00
Alexey Palazhchenko a6c9a8e731 Bump Go and deps (#5236) 2025-06-06 12:44:50 +04:00
Alexey Palazhchenko 74448bee91 Update TODO comments (#5223) 2025-06-05 22:42:55 +04:00
Alexander Tobi Fashakin 7fdf7b333a Add blog post on Mongo Express compatibility (#5194) 2025-06-04 21:03:38 +04:00
Alexey Palazhchenko e10e211fb7 Small documentation tweaks (#5218) 2025-06-04 15:39:34 +04:00
Alexey Palazhchenko 1c8b3650a3 Use Ollama in local setup (#5215) 2025-06-04 12:13:49 +04:00
Alexey Palazhchenko 41a3a30976 Update CTS tool to wrap long lines (#5202) 2025-06-04 05:45:38 +00:00
Alexey Palazhchenko 6a3bb8b22d Disable Markdownlint rule that clashes with MDX (#5210) 2025-06-04 09:35:42 +04:00
Alexey Palazhchenko 04255c2385 Unskip test (#5211) 2025-06-04 09:59:09 +09:00
Alexey Palazhchenko 77d1db5833 Refactor resource tracking tests (#5208) 2025-06-03 15:15:37 +02:00
Alexey Palazhchenko 6231d4efe7 Add test for inserting zero timestamp (#5199) 2025-05-30 10:10:29 +09:00
Alexey Palazhchenko f68073e012 Update DocumentDB (#5201) 2025-05-29 16:55:21 +04:00
Alexey Palazhchenko 92e1c5de00 Update DocumentDB (#5197) 2025-05-28 20:27:57 +04:00
Alexey Palazhchenko 7add88cda3 Do not format MDX as Markdown (#5186) 2025-05-27 20:00:56 +04:00
Alexey Palazhchenko 639484df81 Add permissions to GitHub Actions workflows (#5182) 2025-05-27 08:25:20 +04:00
Alexander Tobi Fashakin a6364fc036 Improve insert operations in guides (#5131)
Closes #5130.
2025-05-26 23:20:31 +04:00
Alexey Palazhchenko a313e974ff Add Dependabot configuration for main-v1 (#5098) 2025-05-26 21:36:07 +04:00
Alexey Palazhchenko 21aeda399f Bump deps (#5170) 2025-05-26 21:12:50 +04:00
Alexey Palazhchenko cb89e5e021 Revert "Update CODEOWNERS" (#5169) 2025-05-26 13:25:39 +04:00
Chi Fujii 2095a33886 Make conninfo a resource (#5166)
Closes #5048.
2025-05-26 08:39:14 +09:00
Balou9 7222581d61 Enable onInlineAuthors in blog settings (#5079)
Closes #4587.
2025-05-23 09:04:30 +09:00
Şahin AKYOL 968d8e5600 Use runtime.Cleanup for resource tracking (#5077)
Closes #5047.
2025-05-22 06:15:36 +00:00
YiandAlexey Palazhchenko a2369501ef Add enableTelemetry config for embedded FerretDB (#5109)
Signed-off-by: jyz0309 <45495947@qq.com>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-05-22 11:37:33 +09:00
Chi Fujii 0858ca0c3b Bump deps (#5159) 2025-05-21 14:57:44 +02:00
Alexander Tobi Fashakin 14feeaaf71 Update tags for next release (#5157) 2025-05-14 17:07:11 +02:00
Alexander Tobi Fashakin 5a94d0f11c Update docker tags in docs (#5156) 2025-05-14 12:31:13 +00:00
Alexey Palazhchenko 4757a853ce Update CODEOWNERS 2025-05-14 16:17:54 +04:00
Alexey Palazhchenko 21fa2fa078 Update versions to point to the next release (#5144) 2025-05-11 15:52:00 +04:00
Alexander Tobi Fashakin 7a4dbf27fd Publish FerretDB v2.2.0 release blog post (#5136) 2025-05-09 22:50:37 +04:00
Alexey Palazhchenko d2d96a7c5e Prepare v2.2.0 release (#5138) 2025-05-09 22:03:55 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko bef234172f Document update of FerretDB image (#5124)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-05-09 17:27:03 +00:00
Alexander Tobi Fashakin 3329903a8f Document Data API usage (#5063)
Closes #4734.
2025-05-09 15:30:17 +00:00
Alexey Palazhchenko 2235b122f5 Update DocumentDB (#5142) 2025-05-09 19:20:02 +04:00
Alexey Palazhchenko 66d2226c83 Update DocumentDB (#5141) 2025-05-09 16:16:07 +04:00
Alexey Palazhchenko 3ccbcf0a8b Remove old TODOs (#5132)
Closes #4726.
2025-05-07 12:42:48 +00:00
Alexey Palazhchenko cc6805ee14 Add full arm64 support (#5113)
Closes #4726.
2025-05-07 16:23:58 +04:00
Patryk KwiatekandAlexey Palazhchenko e1c72fc798 Hook CTS into CI (#5120)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-05-06 16:24:52 +04:00
Patryk Kwiatek f972bbec4a Run CTS tests against Vector Search guide examples (#5122) 2025-05-05 20:54:39 +04:00
Patryk Kwiatek bcd9bbcd10 Run CTS tests against Full Text Search guide examples (#5121) 2025-05-05 17:05:01 +04:00
Patryk Kwiatek eb09b681f0 Run CTS tests against TTL Indexes guide examples (#5123) 2025-05-05 12:37:11 +04:00
Chi Fujii 3bee9cdae9 Decode dropIndexes response (#5117)
Closes #5116.
2025-05-02 14:28:00 +04:00
Vartan Babayan 2396a6caf1 Rename binaries and packages (#5078) 2025-05-01 12:27:04 +00:00
Alexey Palazhchenko b42906983a Update DocumentDB (#5115) 2025-05-01 08:18:14 +04:00
Patryk Kwiatek e1a0d1c6fc Reorgonize cts output files (#5082) 2025-04-25 11:24:20 +00:00
Chi Fujii 383cdf9dac Cleanup evaluation docker image (#5076)
Closes #4976.
2025-04-25 15:14:11 +04:00
Alexey Palazhchenko 908651fc64 Document required features (#5086) 2025-04-25 06:39:42 +04:00
Chi Fujii 8ceec7963e Use volume for state directory (#5083)
Closes #4976.
2025-04-24 16:06:26 +04:00
Alexey Palazhchenko a48f8344be Fix Docker tags for pre-release git tags (#5099) 2025-04-24 12:47:07 +02:00
Alexey Palazhchenko cb364f9c3b Use new wire helpers (#5094) 2025-04-24 00:29:23 +00:00
Alexey Palazhchenko e2f6cf9cf4 Update DocumentDB (#5096) 2025-04-23 16:59:20 +04:00
Alexey Palazhchenko c6a392e062 Revert "Use new wire helpers (#5056)"
This reverts commit 206acd429f.
2025-04-23 10:50:13 +04:00
Alexey Palazhchenko 206acd429f Use new wire helpers (#5056) 2025-04-22 16:11:32 +04:00
Alexey Palazhchenko fb60e03502 Skip bypassEmptyTsReplacement parameters for now (#5080) 2025-04-22 16:10:01 +04:00
Chi Fujii ba5729d60b Unexport handlers (#5084)
Closes #5046.
2025-04-22 12:17:56 +02:00
Chi Fujii c609724261 Use Handle function in clientconn (#5070)
Closes #5046.
2025-04-21 11:14:04 +00:00
Patryk Kwiatek accf86b4df Unconvert syntax examples in documentation (#5072) 2025-04-21 08:36:56 +00:00
Chi Fujii d70dba9c91 Use Handle function in Data API (#5069)
Closes #5046.
2025-04-21 07:20:59 +00:00
Chi Fujii da64b62c31 Define docker tags for ferretdb-eval image (#5068)
Closes #4976.
2025-04-21 10:53:51 +04:00
Chi Fujii 9a79f7e1f4 Implement handler Handle function (#5055)
Closes #5046.
2025-04-17 23:49:52 +00:00
Alexey Palazhchenko 3b8b13f322 Update DocumentDB (#5074) 2025-04-17 18:27:45 +04:00
Chi Fujii 38e619706e Create ferretdb-eval docker image with production build (#5051)
Closes #4976.
2025-04-17 13:30:44 +00:00
Chi Fujii 4ae2259179 Fix docker health check on evaluation image (#5067)
Closes #4976.
2025-04-17 11:21:08 +00:00
Alexander Tobi Fashakin 1739460adc Backport .deb installation guide (#5066) 2025-04-17 12:29:29 +04:00
Alexander Tobi Fashakin 75bc1130b4 Update DocumentDB debian packages (#4959)
Closes #4954.
2025-04-16 20:06:51 +04:00
Alexander Tobi Fashakin 3cad4c3236 Add blog post on migrating to FerretDB with dsync (#5033) 2025-04-16 19:40:24 +04:00
Alexey Palazhchenko 5baca45a6d Update OpenAPI spec description (#5062) 2025-04-16 15:31:24 +00:00
Alexey Palazhchenko a60bba2c20 Rename and move code files in guides (#5057) 2025-04-16 15:20:21 +00:00
Alexey Palazhchenko 0e8fa61a69 Update redirects (#5060) 2025-04-16 17:19:21 +04:00
Patryk Kwiatek 9f7d281d23 Hook CTS tool into documentation building (#4990) 2025-04-16 11:14:56 +00:00
Chi Fujii 459a9fd904 Supervise services in evaluation image (#5041)
Closes #4976.
2025-04-16 15:02:56 +04:00
Alexander Tobi Fashakin 4829440254 Backport Kubernetes docs to 2.1 (#5053) 2025-04-15 23:52:38 +00:00
Alexey Palazhchenko f64f75a60f Update DocumentDB (#5054) 2025-04-15 22:35:29 +04:00
55ddc4bbd4 Remove MongoDB driver v1 (#4961)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
Co-authored-by: Patryk Kwiatek <patryk@kwiatek.xyz>
2025-04-14 16:18:21 +00:00
Chi Fujii 7b5830d298 Rename evaluation image to eval-dev (#5040) 2025-04-14 11:06:04 +00:00
a549d79a4d Add Kubernetes installation guide (#4971)
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-04-11 20:53:30 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko 878728bdae Add blog post on FerretDB and CNPG (#4998)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-04-11 20:25:37 +04:00
Alexey Palazhchenko 503e59c531 Use "PostgreSQL with DocumentDB extension" phrase (#5050) 2025-04-11 19:19:26 +04:00
Alexey Palazhchenko d4e0978b3f Update DocumentDB (#5045) 2025-04-11 15:49:34 +04:00
Alexey Palazhchenko da2006bef6 Update wire (#5042) 2025-04-11 07:04:34 +00:00
Alexey Palazhchenko 1bf9d907a0 Make proxy handler implement an interface (#5036) 2025-04-10 08:12:19 +00:00
Alexey Palazhchenko a2499f6b66 Bump deps (#5027) 2025-04-09 18:52:20 +00:00
Alexey Palazhchenko bbab5c4b6f Move findAndModify integration tests (#5037) 2025-04-09 17:23:37 +00:00
Alexey Palazhchenko 1941560800 Merge Msg and Query into a single type (#5032) 2025-04-08 09:41:02 +04:00
Alexey Palazhchenko fb76e21b64 Minor tweaks for proxy code (#5025) 2025-04-08 09:05:56 +04:00
Alexey Palazhchenko 526554e081 Update deps (#5024) 2025-04-08 09:05:39 +04:00
Alexey Palazhchenko fe980d0bcd Move tests for sessions (#5026) 2025-04-08 09:05:16 +04:00
Alexey Palazhchenko d6757ee63a Update DocumentDB (#5020) 2025-04-07 11:36:11 +04:00
Alexey Palazhchenko 29556167bc Do not send zero values to telemetry (#5016) 2025-04-07 09:49:54 +04:00
Alexey Palazhchenko a89ba41c53 Update Docker images (#5015) 2025-04-04 14:11:51 +04:00
Alexander Tobi Fashakin ffe987aa75 Add blogpost on FerretDB v2.1.0 release (#5004) 2025-04-03 22:12:41 +04:00
Alexey Palazhchenko 05ed2b952c Prepare v2.1.0 release (#5013) 2025-04-03 19:34:04 +04:00
Alexey Palazhchenko 1187aac976 Update changelog generator (#4810) 2025-04-03 18:53:51 +04:00
Alexey Palazhchenko 1630cbd4a2 Disable commit check for now (#5012) 2025-04-03 14:11:34 +04:00
Alexey Palazhchenko d95475ba57 Create redirects for /v2.0/ documentation URLs (#5011) 2025-04-03 13:21:22 +04:00
Alexey Palazhchenko f78160a69d Remove error middleware for now (#5008)
Closes #4965.
2025-04-03 08:44:01 +04:00
Alexey Palazhchenko 5f890da6ac Use Go 1.24.2 (#5005) 2025-04-02 17:11:16 +04:00
Alexey Palazhchenko 6f86e9098d Partially revert middleware changes (#4995) 2025-04-02 11:42:06 +04:00
Alexey Palazhchenko b962266a1a Fix version detection for embeddable package (#4936)
Closes #4889.
2025-04-01 18:18:46 +04:00
Alexey Palazhchenko 3ff4172ee9 Add more tests for error handling (#4996) 2025-04-01 07:18:35 +04:00
Alexander Tobi Fashakin 869b4cf727 Update feature blog post (#4991) 2025-03-27 21:41:15 +04:00
Alexey Palazhchenko b1277aea38 Add basic documentation for supported commands (#4970) 2025-03-27 21:38:16 +04:00
a520043a58 Add post on MongoDB queries and operations on FerretDB (#4732)
Closes FerretDB/engineering#235.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
2025-03-27 21:20:48 +04:00
Alexey Palazhchenko f320ffb026 Do not decode incoming document twice (#4981) 2025-03-27 17:54:26 +04:00
Alexey Palazhchenko c3a8118d86 Add TODOs (#4986) 2025-03-27 11:22:06 +04:00
Alexey Palazhchenko 2c0cadf502 Update DocumentDB (#4982) 2025-03-26 13:16:12 +04:00
Alexey Palazhchenko 09eddc89b2 Disable tracing in benchmarks for now (#4980)
Closes #4934.
2025-03-26 11:14:00 +04:00
Alexey Palazhchenko f501409464 Update DocumentDB (#4975) 2025-03-25 13:58:00 +04:00
Alexey PalazhchenkoandChi Fujii 5d7626b627 Update DocumentDB (#4941)
Closes #4966.

Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
2025-03-25 12:26:56 +04:00
Alexey Palazhchenko 077b782f31 Use ERROR level logging for failed tests (#4974) 2025-03-25 10:58:50 +04:00
Chi Fujii 82604f8b51 Implement error middleware (#4951)
Closes #1997.
2025-03-24 23:39:25 +04:00
Alexey Palazhchenko 9c1ce6280e Backport documentation changes to v2.0 (#4952) 2025-03-21 16:42:53 +04:00
Alexander Tobi Fashakin 4d0e2eb47f Update deb and rpm installation docs (#4927) 2025-03-21 10:35:32 +00:00
krishna sindhur a3cba3361d Add support for reading PostgreSQL URL from a file (#4937)
Closes #4877.
2025-03-21 10:00:49 +00:00
Alexey Palazhchenko fc64cd0de5 Update/backport full-text and TTL indexes guides (#4948) 2025-03-21 01:03:48 +00:00
Chi Fujii 6e03fb1b0a Handle OP_QUERY in middleware using interface (#4947)
Closes #1997.
2025-03-20 23:39:50 +04:00
Alexey Palazhchenko d89cfc980f Add a note to documentation about PR titles (#4944) 2025-03-20 00:18:22 +00:00
Alexander Tobi Fashakin b444c5a778 Add TTL index guide (#4926) 2025-03-19 19:06:17 +04:00
Alexey Palazhchenko df3c5d52ce Improve integration benchmarks (#4925) 2025-03-18 21:15:22 +04:00
Alexey PalazhchenkoandChi Fujii fa3d68f682 Change the way OP_MSG handlers are invoked (#4923)
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
2025-03-18 21:12:32 +04:00
Alexey Palazhchenko 150ecb8186 Sync flags grouping with docs (#4933)
Closes #4746.
2025-03-18 15:11:00 +04:00
krishna sindhur 96bb7392cd refactored: help command flags into groups (#4918) 2025-03-17 17:01:23 +04:00
Alexey Palazhchenko 2809469f2a Configure connection pool size in tests (#4932) 2025-03-17 10:50:04 +04:00
Alexander Tobi Fashakin 9a706578a2 Add guide for Compass as a compatible app (#4906) 2025-03-14 19:54:09 +04:00
Patryk Kwiatek e814886570 Add colorized levels to console logger (#4904) 2025-03-14 00:17:17 +00:00
Chi Fujii 1b78786599 Move message processing in clientconn to a function (#4919)
Closes #1997.
2025-03-14 00:07:38 +00:00
Alexander Tobi Fashakin f0bcec5e09 Add full text search guide (#4886) 2025-03-13 23:03:52 +04:00
Alexey Palazhchenko b2816b362f Add basic structure for middlewares (#4908) 2025-03-13 12:22:00 +04:00
Alexey Palazhchenko 553741c8ed Bump deps (#4916) 2025-03-13 12:21:23 +04:00
Chi Fujii ec52fef14b Refactor clientconn (#4907)
Closes #1997.
2025-03-12 19:46:42 +04:00
Alexey Palazhchenko 125e16c240 Make production builds of the main branch (#4911) 2025-03-12 05:43:23 +04:00
Alexey Palazhchenko fd460f08fe Add TODO comments for observability tasks (#4896) 2025-03-10 16:15:55 +04:00
Alexey Palazhchenko c1e0a595ba Add example telemetry report to documentation (#4861)
Closes #4721.
2025-03-08 11:35:15 +00:00
Chi Fujii a5eb60bdb4 Check issue URLs in documentation (#4890)
Closes #4715.
2025-03-07 12:20:54 +01:00
Patryk Kwiatek 27b210bbda Update TODO URLs for listDatabase tests (#4863)
Closes #4722.
2025-03-07 14:49:54 +04:00
Patryk Kwiatek c7ee74e83c Document non-enforced format of log messages in envtool package (#4867) 2025-03-07 10:51:59 +04:00
Alexey Palazhchenko ebd085685f Start working on a new release (#4885) 2025-03-06 22:32:57 +04:00
Alexey Palazhchenko fb4d59f6a9 Adjust pool connection timeout (#4888)
Closes #4876.
2025-03-06 10:21:18 +04:00
Alexey Palazhchenko a9c0b5bd5f Add FerretDB v2 GA blog post (#4884) 2025-03-05 19:08:29 +04:00
Alexey Palazhchenko 2214721e51 Prepare 2.0.0 GA release! (#4881) 2025-03-05 16:26:16 +04:00
Alexey Palazhchenko 84fc64c1a6 Final preparations (#4874) 2025-03-05 15:47:35 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko aad81dcf9d Add basic troubleshooting guide (#4854)
Closes #4844.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-03-05 14:48:21 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko 2c9143eb37 Add migration guide from v1.x to v2.x (#4850)
Closes #4789.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-03-05 14:27:06 +04:00
Alexey Palazhchenko 077887e634 Bump Go and safe deps (#4875) 2025-03-05 10:07:54 +04:00
Chi Fujii 14e72a10b2 Skip flaky currentOp test and add TODO (#4872) 2025-03-05 09:12:01 +04:00
Alexey Palazhchenko cebdfb845e Use GitHub-hosted CI runners (#4866) 2025-03-04 21:53:11 +04:00
Alexey Palazhchenko 4051629d92 Unskip tests that refer to closed issue (#4865) 2025-03-04 20:28:24 +04:00
Alexey Palazhchenko 224ad9242b Prepare release (#4859) 2025-03-04 16:45:04 +04:00
Alexander Tobi FashakinandAlexey Palazhchenko ae554b683f Recommend using full tags/versions (#4834)
Closes #4833.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-03-04 15:37:39 +04:00
Alexey Palazhchenko f32f80a771 Improve messages about DocumentDB version (#4832)
Closes #4724.
2025-03-04 15:08:06 +04:00
Chi FujiiandAlexey Palazhchenko 942ee8d3b4 Implement currentOp command (#4855)
Closes #4794.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-03-04 14:31:42 +04:00
Chi FujiiandAlexey Palazhchenko 5a144ed783 Remove DocumentDB building from this repo (#4852)
Closes #4725.

Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-03-04 13:25:25 +04:00
Alexey PalazhchenkoandPatryk Kwiatek 64ff290f79 Use new DocumentDB API for listDatabases (#4841)
Closes #4722.

Co-authored-by: Patryk Kwiatek <patryk.kwiatek@ferretdb.io>
2025-03-04 12:45:08 +04:00
Alexey PalazhchenkoandChi Fujii ee32bc86ed Remove old code for tracking operations (#4842)
Closes #4794.

Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
2025-03-04 12:43:15 +04:00
Chi FujiiandAlexey Palazhchenko 7773990627 Update DocumentDB Docker image (#4853)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-03-04 11:50:29 +04:00
Alexey Palazhchenko 0882beff7b Update URL to join Slack (#4851) 2025-03-03 15:36:24 -05:00
a0936cb7e6 Add Data API blogpost (#4739)
Closes FerretDB/engineering#237.

Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-02-28 20:33:37 +04:00
Alexey Palazhchenko de87a39474 Make Data API without authentication if requested (#4849) 2025-02-28 20:24:25 +04:00
Alexey Palazhchenko 044ab23b9e Filter out sensitive information from debug archive (#4836)
`cmdline` may include credentials in PostgreSQL URL, etc.

Closes #4838.
2025-02-28 11:24:31 +00:00
Alexey Palazhchenko da89bec0d3 Move comment in docs to fix DocCard (#4845) 2025-02-28 11:07:48 +00:00
Patryk Kwiatek e61766b6db Enforce log messages format for debug build and integration tests (#4754) 2025-02-28 08:09:26 +04:00
Alexey Palazhchenko 375b23c4e5 Publush build info, state, and CLI flags in expvar (#4837)
Those expvars are added to the debug archive that we could ask users to give us for debugging.

Closes #4838.
2025-02-27 13:55:48 +01:00
Alexey Palazhchenko 584dde053c Bump DocumentDB for development (#4839)
Unblocks `currentOp` implementation.

Closes #4794.
2025-02-27 13:32:46 +01:00
Alexey Palazhchenko a6806efd1d Update TODO comment (#4831) 2025-02-26 17:07:06 +04:00
Patryk KwiatekandAlexey Palazhchenko 84b3efd41c Change dbStats tests TODO links (#4823)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-02-26 09:38:28 +04:00
6267c4eb64 Add blog post for rc2 release (#4817)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-02-25 22:35:41 +04:00
Alexey Palazhchenko eb1ef1ed2c Prepare v2.0.0-rc.2 release (#4816) 2025-02-25 21:43:26 +04:00
Alexander Tobi Fashakin d170db2b07 Update blogs to use js instead of json5 (#4801) 2025-02-24 18:29:39 +04:00
Alexey Palazhchenko 1453898cbc Enable Docker Hub and quay.io (#4815) 2025-02-24 18:16:31 +04:00
Alexey Palazhchenko 6c958980bb Update Docker tags (#4812) 2025-02-24 17:45:54 +04:00
Alexey Palazhchenko c927df3df1 Update DocumentDB (#4811) 2025-02-24 17:19:35 +04:00
Alexey Palazhchenko 840308b6b4 Enable building FerretDB as PostgreSQL background worker (#4802) 2025-02-24 13:23:57 +04:00
Patryk Kwiatek 04de99a54d Implement dbStats command (#4804) 2025-02-24 13:20:20 +04:00
Alexey PalazhchenkoandCopilot b330166403 Fix envtool for Go 1.24 (#4809)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-02-24 13:07:18 +04:00
Chi FujiiandAlexey Palazhchenko 210cafb4b5 Update release checklist and .deb installation guide (#4767)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-02-24 11:40:51 +04:00
Alexey Palazhchenko e8bb57932b Make embeddable package logging configurable (#4807) 2025-02-24 09:58:54 +04:00
Alexey Palazhchenko 3a9706658a Tweak Docusaurus configuration (#4805) 2025-02-24 09:38:03 +04:00
Alexey Palazhchenko 39c4ab5552 Re-introduce embeddable package (#4778) 2025-02-21 14:26:47 +00:00
Alexey Palazhchenko e3039ac2c9 Bump Go deps (#4803) 2025-02-21 14:45:53 +04:00
Patryk Kwiatek 04c2cfb016 Add mongo slog format (#4716) 2025-02-21 06:49:35 +00:00
Alexander Tobi Fashakin ddbfc7260b Convert json5 to js responses with single quotes (#4800) 2025-02-21 05:57:20 +00:00
Alexey Palazhchenko 63c1f21a22 Add test helper (#4799) 2025-02-21 09:50:11 +04:00
Alexey Palazhchenko 174b472cbc Make handler take over the pool (#4795) 2025-02-21 09:49:52 +04:00
Alexey Palazhchenko cfe9f56456 Update Docker image tag in v1 docs (#4796) 2025-02-20 19:23:23 +04:00
Alexey Palazhchenko 6f38b9a26b Reformat request/response .js files (#4787) 2025-02-20 18:49:04 +04:00
Alexey PalazhchenkoandChi Fujii ca39e0c41f Update DocumentDB (#4788)
Co-authored-by: Chi Fujii <chi.fujii@ferretdb.io>
2025-02-20 11:50:17 +04:00
Alexey Palazhchenko c1f0fef933 Use directories instead of files for state and telemetry (#4784) 2025-02-19 10:05:15 +09:00
Alexey Palazhchenko 6adc30a2bc Tweak CLI for disabling interfaces (#4781) 2025-02-18 09:32:25 -05:00
Alexey Palazhchenko f812de58e1 Use Go 1.24.0 (#4775) 2025-02-17 19:20:13 +04:00
Alexey Palazhchenko d94e91b1fd Consistently use / in MongoDB URIs (#4759) 2025-02-13 02:31:10 +00:00
Alexey Palazhchenko 6bc29eb0b6 Set GOARM64 explicitly (#4760) 2025-02-13 00:34:18 +00:00
Alexey Palazhchenko 00aff27c5c Remove golang.org/x/exp/maps package (#4711) 2025-02-13 00:25:21 +00:00
Alexey Palazhchenko 524394fafb Add/update TODO comments for some issues (#4723) 2025-02-12 15:28:21 +01:00
Alexander Tobi Fashakin 04f5c06df0 Update TLS docs (#4752) 2025-02-12 06:36:44 +00:00
Alexander Tobi Fashakin 9b4720b550 Update config flags (#4740) 2025-02-12 06:27:56 +00:00
Chi Fujii e561e88d78 Implement reIndex command (#4735)
Closes #1516.
2025-02-12 10:20:14 +04:00
Tomáš Linhart a9c4b8c639 Add missing space in command (#4753) 2025-02-11 12:45:02 +04:00
Alexey Palazhchenko b085d709b5 Remove "all backends" from the issue template (#4751) 2025-02-11 00:29:03 +00:00
Alexey Palazhchenko ae1d2ccfea Add explicit platform: linux/amd64 to docs (#4749) 2025-02-10 21:17:17 +00:00
Alexander Tobi Fashakin 95eb803046 Add replication blog post (#4717) 2025-02-10 18:29:05 +04:00
Chi Fujii aac25390a7 Use wireclient for readiness probe to avoid calling endsessions on unauthenticated client (#4718)
Closes #4692.
2025-02-10 12:39:17 +00:00
Chi FujiiandAlexey Palazhchenko f569ea500b Check DocumentDB issues in checkcomments and checkdocs (#4714)
Co-authored-by: Alexey Palazhchenko <alexey.palazhchenko@ferretdb.io>
2025-02-10 16:29:47 +04:00
Chi Fujii 06697e2db9 Refactor and generate using genwrap tool (#4705)
Closes #4700.
2025-02-10 09:38:10 +00:00
Alexey Palazhchenko 2976b96d70 Bump deps (#4743) 2025-02-07 17:05:58 +04:00
Alexey Palazhchenko 0cf6ca0344 Do not use a global variable for logging (#4727) 2025-02-03 01:18:14 +00:00
Patryk Kwiatek 8cfa10ebc1 Introduce smoke test for Data API (#4712) 2025-02-02 07:12:19 +01:00
Chi Fujii fc7d64802e Refactor extract and convert in genwrap tool (#4703)
Closes #4700.
2025-01-29 05:58:33 +00:00
Alexey Palazhchenko adc44a2f03 Do not use Git LFS for Go files (#4713) 2025-01-29 09:50:45 +04:00
Patryk Kwiatek ff0ac6e749 Fix error mapping generation (#4691) 2025-01-29 05:42:10 +00:00
Alexey Palazhchenko bc19b5b451 Fix Dependabot config (#4708) 2025-01-28 14:49:15 +04:00
Chi Fujii dd0ab00897 Skip private issue links in checkcomments by a flag (#4695) 2025-01-28 12:39:12 +04:00
Alexey Palazhchenko 9305598b5f Disable xfail CI configuration for now (#4698) 2025-01-27 10:11:52 +04:00
Alexey Palazhchenko 279cbfd9b3 Tweak documentation 2025-01-23 21:12:41 +04:00
Alexey Palazhchenko a289181298 Fix Docker image names 2025-01-23 20:59:51 +04:00
Alexey Palazhchenko f69b9a10b5 Revert docs build changes 2025-01-23 20:45:11 +04:00
Alexey Palazhchenko 35095d7c0e Skip Docker Hub and Quay.io for now 2025-01-23 20:40:52 +04:00
Alexey Palazhchenko e7173c3d8d Add FerretDB v2 (#4689) 2025-01-23 20:29:50 +04:00
1209 changed files with 81258 additions and 70708 deletions

No files matched your search

+1
View File
@@ -15,3 +15,4 @@
# additional files for Docker images
!build/ferretdb
!build/postgres-documentdb
+9
View File
@@ -0,0 +1,9 @@
# https://editorconfig.org
root = true
[*]
end_of_line = lf
trim_trailing_whitespace = true
insert_final_newline = true
charset = utf-8
+5
View File
@@ -3,3 +3,8 @@
*.png filter=lfs diff=lfs merge=lfs -text
*.webp filter=lfs diff=lfs merge=lfs -text
*.gif filter=lfs diff=lfs merge=lfs -text
*.pdf filter=lfs diff=lfs merge=lfs -text
*.go diff=golang
.gitea/CODEOWNERS linguist-detectable=false
+3
View File
@@ -0,0 +1,3 @@
# Pull requests request review from the eligible staff team(s).
# One approval from either an Admin or Developer satisfies branch protection.
* @trueog/Admins @trueog/Developers
+4 -15
View File
@@ -1,21 +1,10 @@
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
# Order is important; the last matching pattern takes the most precedence.
# Order is important; the last matching pattern has the highest precedence.
* @AlekSi
* @AlekSi @chilagrow
/cmd/ @FerretDB/leads
/ferretdb/ @FerretDB/leads
/integration/ @FerretDB/leads
/internal/ @FerretDB/leads
/internal/backends/ @AlekSi
/website/ @AlekSi @Fashander
# @AlekSi for engineering, @ptrfarkas for marketing.
# See also Mergify configuration.
/website/ @AlekSi @ptrfarkas
/build/ @AlekSi
/build/legacy-mongo-shell/ @FerretDB/specops
go.mod @AlekSi
go.sum @AlekSi
/website/blog/ @AlekSi @ptrfarkas
+15 -19
View File
@@ -14,18 +14,14 @@ body:
- type: textarea
id: version
attributes:
label: FerretDB version
label: FerretDB, PostgreSQL, and DocumentDB versions
description: >
What version and package of FerretDB do you use?
Please provide an output of running FerretDB with [`--version` flag](https://docs.ferretdb.io/v1.24/configuration/flags/).
validations:
required: true
- type: input
id: backend
attributes:
label: Backend
description: >
What backend (PostgreSQL, etc) and what version do you use?
Please add the output of the `serverStatus` command.
value: |
```json
{
}
```
validations:
required: true
- type: textarea
@@ -33,13 +29,13 @@ body:
label: Environment
description: |
For example:
- **OS**: Ubuntu 22.04 x86_64
- **Deployment**: Docker using `ghcr.io/ferretdb/ferretdb:1.15.0` image
- **Deployment details**: Docker version 24.0.6
- **OS**: Ubuntu 24.04 x86_64
- **Deployment**: Docker using `ghcr.io/ferretdb/ferretdb:2` image
- **Deployment details**: Docker version 27.3.1
value: |
- OS:
- Deployment:
- Deployment details:
- **OS**:
- **Deployment**:
- **Deployment details**:
validations:
required: true
- type: textarea
@@ -48,7 +44,7 @@ body:
label: What did you do?
description: >
Include MongoDB queries or
[pull request with a test script](https://github.com/FerretDB/FerretDB/blob/main-v1/CONTRIBUTING.md#reporting-a-bug)
[pull request with a test script](https://github.com/FerretDB/FerretDB/blob/main/CONTRIBUTING.md#reporting-a-bug)
if you can.
validations:
required: true
@@ -65,6 +61,6 @@ body:
attributes:
label: What did you see instead?
description: >
Please include FerretDB [logs](https://docs.ferretdb.io/v1.24/configuration/observability/).
Please include FerretDB [logs](https://docs.ferretdb.io/configuration/observability/).
validations:
required: true
+1 -2
View File
@@ -31,11 +31,10 @@ body:
description: What should be done to consider this issue done? List everything that applies.
value: |
- handler updated;
- all backends updated;
- unit tests added/updated;
- integration/compatibility tests added/updated;
- spot refactorings done;
- user documentation updated or an issue to create documentation created;
- user documentation updated (or an issue to create documentation created);
- something else?
validations:
required: true
+1 -1
View File
@@ -4,7 +4,7 @@
blank_issues_enabled: false
contact_links:
- name: Quick question
url: https://join.slack.com/t/ferretdb/shared_invite/zt-zqe9hj8g-ZcMG3~5Cs5u9uuOPnZB8~A
url: https://slack.ferretdb.io/
about: Please join our Slack chat.
- name: General question or discussion
url: https://github.com/FerretDB/FerretDB/discussions/categories/general
+5 -3
View File
@@ -10,8 +10,8 @@ This guide tries to be short and does not mention things that are completely aut
We want our work to be visible and transparent.
This way we can help each other and make sure that we are on the right track.
To uphold these principles, we agree to make results of our daily work available for others through various means such as
pull requests, code reviews, created issues, or any other relevant method.
To uphold these principles, we agree to make results of our daily work available for others through various means
such as pushed commits, updated pull requests, code reviews, created issues, or any other relevant method.
### Community
@@ -27,8 +27,10 @@ please follow these notes below when you working with pull request:
1. Send pull requests from forks; do not make personal branches in the main repository.
This way, we are similar to community members and could notice similar problems that we could fix for everyone,
not just for us.
2. Pull request title should be accurate and descriptive as it is used in the generated changelog.
2. Pull request title should be accurate and descriptive
as it is used in the generated [changelog](../CHANGELOG.md) as is.
It generally should start with imperative verb ("Fix …", not "Fixing …", "Fixed …" or other forms).
It should not have a prefix ("Fix …", not "Fix: …") (we use labels for categorization).
It should not mention the issue number.
3. It is fine to send several sequential pull requests for one issue to make them easier to review.
In that case, please still use the `Closes` word as described in [CONTRIBUTING.md](../CONTRIBUTING.md)
+38 -23
View File
@@ -2,21 +2,30 @@
## Preparation
1. Check tests, linters.
2. Check issues and pull requests, update milestones and labels.
3. Check if there are breaking changes or anything that should be reflected in the changelog and/or user documentation.
4. Create draft release on GitHub with autogenerated release notes.
5. Generate changelog:
- Run `task changelog MILESTONE_TITLE=vX.Y.Z PREVIOUS_MILESTONE_TITLE=vX.Y.Z` to generate release changelog for the given milestone.
- Copy the changelog to the `CHANGELOG.md` file.
1. [Release DocumentDB](https://github.com/FerretDB/documentdb/blob/ferretdb/.github/RELEASE_CHECKLIST.md).
2. Update DocumentDB image tags to point to release in:
- [eval image](../build/ferretdb/eval.Dockerfile);
- [eval-dev image](../build/ferretdb/eval-dev.Dockerfile);
- [local image](../build/deps/postgres-documentdb.Dockerfile);
- [bw image](../build/ferretdb-bw/Dockerfile).
3. Update constants in [`documentdb.go`](../build/version/documentdb.go).
4. Check DocumentDB images/tags in documentation (where we specify full tags).
5. Re-run tests, linters.
6. Check issues and pull requests, update milestones and labels.
7. Check if there are breaking changes or anything that should be reflected in the changelog and/or user documentation.
8. Create draft release on GitHub with autogenerated release notes.
9. Update CHANGELOG.md:
- Run `task changelog PREV=vX.Y.Z NEXT=vX.Y.Z` to generate release changelog for the given milestone.
- Check warnings, copy output to the `CHANGELOG.md` file.
- Sort items within sections according to importance if needed.
- Add first-time contributor credits if any (copy this data from the release draft).
6. Run `task docs-version VERSION=X.Y`.
Update `versions` in `docusaurus.config.js`.
Remove the oldest version from `versioned_docs`, `versioned_sidebars`, `versions.json`.
7. Run `task docs-fmt`.
8. Commit and push changes to the PR.
9. Merge PR, pull and check `git status`.
- Add link to DocumentDB release page.
10. Run `task docs-version VERSION=X.Y`.
Update `versions` and `createRedirects` in `docusaurus.config.js`.
Remove the oldest version from `versioned_docs`, `versioned_sidebars`, `versions.json`.
11. Run `task docs`.
12. Commit and push changes to the PR.
13. Merge PR, pull and check `git status`.
## Git tag
@@ -24,8 +33,8 @@
2. Check `task gen-version; git status` output.
3. Push it!
4. Refresh
- `env GOPROXY=https://proxy.golang.org go mod download -x github.com/FerretDB/FerretDB@<vX.Y.Z>`
- `https://pkg.go.dev/github.com/FerretDB/FerretDB@<vX.Y.Z>` from https://pkg.go.dev/github.com/FerretDB/FerretDB?tab=versions.
- `env GOPROXY=https://proxy.golang.org go mod download -x github.com/FerretDB/FerretDB/v2@<vX.Y.Z>`
- `https://pkg.go.dev/github.com/FerretDB/FerretDB/v2@<vX.Y.Z>` from https://pkg.go.dev/github.com/FerretDB/FerretDB/v2?tab=versions.
## Release
@@ -36,21 +45,27 @@
4. Check:
- https://hub.docker.com/r/ferretdb/ferretdb/tags
- https://hub.docker.com/r/ferretdb/ferretdb-dev/tags
- https://hub.docker.com/r/ferretdb/all-in-one/tags
- https://hub.docker.com/r/ferretdb/ferretdb-eval/tags
- https://hub.docker.com/r/ferretdb/ferretdb-eval-dev/tags
- https://github.com/FerretDB/FerretDB/pkgs/container/ferretdb
- https://github.com/FerretDB/FerretDB/pkgs/container/ferretdb-dev
- https://github.com/FerretDB/FerretDB/pkgs/container/all-in-one
- https://github.com/FerretDB/FerretDB/pkgs/container/ferretdb-eval
- https://github.com/FerretDB/FerretDB/pkgs/container/ferretdb-eval-dev
- https://quay.io/repository/ferretdb/ferretdb?tab=tags
- https://quay.io/repository/ferretdb/ferretdb-dev?tab=tags
- https://quay.io/repository/ferretdb/all-in-one?tab=tags
- https://quay.io/repository/ferretdb/ferretdb-eval?tab=tags
- https://quay.io/repository/ferretdb/ferretdb-eval-dev?tab=tags
5. Close milestone in issues.
6. Publish release on GitHub.
7. Announce it on Slack.
## Soon after
1. Bump the latest version on https://beacon.ferretdb.com and https://beacon.ferretdb.io.
2. Publish and announce blog post.
3. Tweet, toot.
4. Update NixOS package: https://github.com/NixOS/nixpkgs/tree/master/pkgs/servers/nosql/ferretdb.
5. Update Civo package: https://github.com/civo/kubernetes-marketplace/tree/master/ferretdb.
1. Publish and announce blog post.
2. Add link to blog post to `CHANGELOG.md` and release page.
3. Bump the latest version on https://beacon.ferretdb.com and https://beacon.ferretdb.io.
4. Update DocumentDB images/tags in Dockerfiles to point back to development version.
5. Update DocumentDB images/tags in the current documentation (where we specify full tags)
to point to the next (non-existing) version.
6. Update URLs with `/refs/tags/` in them in the current documentation
to point to the next (non-existing) version.
+26 -1
View File
@@ -30,7 +30,32 @@ flag_management:
# https://docs.codecov.com/docs/pull-request-comments
comment:
layout: reach,diff,files,flags
layout: reach,diff,files,flags,components
# https://docs.codecov.com/docs/components
component_management:
default_rules:
statuses:
- type: project
target: auto
threshold: 1%
individual_components:
- component_id: clientconn
name: "clientconn"
paths:
- internal/clientconn/**
- component_id: dataapi
name: "dataapi"
paths:
- internal/dataapi/**
- component_id: documentdb
name: "documentdb"
paths:
- internal/documentdb/**
- component_id: handler
name: "handler"
paths:
- internal/handler/**
# https://docs.codecov.com/docs/github-checks
# Annotations may be hidden by pressing the "a" key or by unselecting "Show annotations" in the top right of the file.
+122
View File
@@ -3,6 +3,8 @@
# Apply the "not ready" label to prevent cascaded CI runs on each auto-merge.
version: 2
# Most configurations are duplicated to add main-v1 branch.
updates:
- package-ecosystem: "github-actions"
directory: "/"
@@ -16,6 +18,19 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "github-actions"
target-branch: "main-v1"
directory: "/"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
github-actions-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "docker"
directory: "/build/ferretdb"
@@ -29,6 +44,20 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "docker"
target-branch: "main-v1"
directory: "/build/ferretdb"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
build-docker-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "docker"
directory: "/build/deps"
labels: ["deps", "not ready"]
@@ -41,6 +70,19 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "docker"
target-branch: "main-v1"
directory: "/build/deps"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
build-deps-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
directory: "/"
@@ -54,6 +96,20 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
target-branch: "main-v1"
directory: "/"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
ferretdb-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
directory: "/integration"
labels: ["deps", "not ready"]
@@ -66,6 +122,20 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
target-branch: "main-v1"
directory: "/integration"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
integration-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
directory: "/tools"
labels: ["deps", "not ready"]
@@ -78,6 +148,20 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
target-branch: "main-v1"
directory: "/tools"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
tools-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
directory: "/tools/golangci"
labels: ["deps", "not ready"]
@@ -90,3 +174,41 @@ updates:
interval: "weekly"
day: "wednesday"
time: "01:42"
- package-ecosystem: "gomod"
target-branch: "main-v1"
directory: "/tools/golangci"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
golangci-v1:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
# no mcphost in main-v1
- package-ecosystem: "gomod"
directory: "/tools/mcphost"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
groups:
mcphost:
patterns: ["*"]
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
# no submodules in main-v1
- package-ecosystem: "gitsubmodule"
directory: "/"
labels: ["deps", "not ready"]
assignees: [AlekSi]
open-pull-requests-limit: 20
schedule:
interval: "weekly"
day: "wednesday"
time: "01:42"
-1
View File
@@ -11,7 +11,6 @@ pull_request_rules:
- "-draft"
- "-locked"
- "-merged"
- "author != dependabot[bot]"
- "base = main"
- "branch-protection-review-decision = APPROVED"
- "check-success = golangci-lint"
-33
View File
@@ -1,33 +0,0 @@
---
# https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes
changelog:
categories:
- title: New Features 🎉
labels:
- code/feature
- title: Fixed Bugs 🐛
labels:
- code/bug
- code/bug-regression
- title: Enhancements 🛠
labels:
- code/enhancement
- title: Documentation 📄
labels:
- blog/engineering
- blog/marketing
- documentation
- title: Other Changes 🤖
labels:
- code/chore
- project
- deps
- title: This section should be empty
labels:
- "*"
+12 -22
View File
@@ -12,6 +12,7 @@ repository:
enable_vulnerability_alerts: true
# https://docs.github.com/en/rest/issues/labels
# Labels are being used by issues, PRs, and discussions.
labels:
# area
- name: area/admin
@@ -29,44 +30,33 @@ labels:
- name: area/cursors
color: "#D93F0B"
description: Issues about cursors
- name: area/dataapi
color: "#D98F93"
description: Issues about Data API
- name: area/diag
color: "#909BA6"
description: Issues about diagnostic commands
- name: area/errors
color: "#DE231F"
description: Issues about MongoDB-compatible error codes and messages
- name: area/fuzz
color: "#D4C5F9"
description: Issues about fuzzing and smithing
- name: area/indexes
color: "#006B75"
description: Issues about indexes
- name: area/mcp
color: "#95921F"
description: Issues about MCP
- name: area/observability
color: "#E04D33"
description: Issues about metrics, logs, traces, pprof, etc
- name: area/oplog
color: "#09D521"
description: Issues about OpLog and capped collections
- name: area/params
color: "#FBCA04"
description: Issues about command parameters extraction
- name: area/performance
color: "#D02FE8"
description: Issues about performance
- name: area/pushdown
color: "#83F7BA"
description: Issues about query pushdown
- name: area/types
- name: area/wire
color: "#0354F4"
description: Issues about data types, BSON, wire protocol, etc
# backend
- name: backend/pg
color: "#305C83"
description: Issues about PostgreSQL backend
- name: backend/sqlite
color: "#2C8FCA"
description: Issues about SQLite backend
- name: backend/hana
color: "#2D6AB1"
description: Issues about SAP HANA backend
description: Issues about BSON and the wire protocol
# blog
- name: blog/engineering
+68 -39
View File
@@ -12,24 +12,18 @@ on:
shard_total:
required: true
type: number
coveralls:
required: true
type: boolean
disable_pushdown:
test_timeout:
required: false
type: string
default: 20m
no_xfail:
required: false
type: boolean
default: false
postgres_dockerfile:
required: false
type: string
default: "postgres"
checkout_ref:
required: false
type: string
default: ""
secrets:
FERRETDB_HANA_URL:
skip_flaky_check:
required: false
type: boolean
default: false
env:
GOPATH: /home/runner/go
@@ -43,30 +37,20 @@ jobs:
run:
# make it short to fit in GitHub UI; all parameters are already in the caller's name
name: Run
# https://www.ubicloud.com/docs/about/pricing#github-actions
# https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates-for-larger-runners
runs-on: ubicloud-standard-4
runs-on: ubuntu-24.04
timeout-minutes: 25
steps:
- name: Dump github context
run: echo "$GITHUB_CONTEXT"
env:
GITHUB_CONTEXT: ${{ toJSON(github) }}
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
ref: ${{ inputs.checkout_ref }}
# for branch.txt on pull_request_target; the correct branch is already checked out on push / schedule
# for branch.txt on pull_request; the correct branch is already checked out on push / schedule
- name: Name branch
if: github.event_name == 'pull_request_target'
if: github.event_name == 'pull_request'
env:
BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
run: git checkout -b $BRANCH
@@ -80,55 +64,100 @@ jobs:
run: go generate -x
working-directory: tools
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Start environment
run: bin/task env-up-detach
env:
POSTGRES_DOCKERFILE: ${{ inputs.postgres_dockerfile }}
- name: Run init
run: bin/task init
- name: Setup Ollama and MCPHost
run: |
mkdir -p tmp/logs
curl -fsSL https://ollama.com/install.sh | sh
sudo systemctl stop ollama
bin/task ollama > tmp/logs/ollama.log 2>&1 &
bin/task mcphost > tmp/logs/mcphost.log 2>&1 &
- name: Wait for and setup environment
run: bin/task env-setup
- name: >
Run ${{ inputs.task }} tests
(${{ inputs.shard_index }}/${{ inputs.shard_total }},
pushdown=${{ !inputs.disable_pushdown }})
(${{ inputs.shard_index }}/${{ inputs.shard_total }})
run: >
bin/task test-integration-${{ inputs.task }}
SHARD_INDEX=${{ inputs.shard_index }}
SHARD_TOTAL=${{ inputs.shard_total }}
TEST_TIMEOUT=20m
DISABLE_PUSHDOWN=${{ inputs.disable_pushdown }}
TEST_TIMEOUT=${{ inputs.test_timeout }}
NO_XFAIL=${{ inputs.no_xfail }}
env:
GOFLAGS: ${{ runner.debug == '1' && '-v' || '' }}
FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }}
ENVTOOL_DEBUG: ${{ runner.debug == '1' && 'true' || 'false' }}
- name: Collect logs
if: failure()
run: |
bin/task env-logs-collect
cp -v integration/*.json.tmp integration/*.txt.tmp tmp/logs/
- name: Compress logs before upload
if: failure()
run: zip -r -q -9 logs.zip tmp/logs
- name: Upload logs
if: failure()
uses: actions/upload-artifact@v5
with:
name: logs-${{ inputs.task }}-${{ inputs.shard_index }}-${{ inputs.shard_total }}
path: logs.zip
retention-days: 3
- name: Upload coverage information to codecov
if: always()
uses: codecov/codecov-action@v5
with:
files: ./integration/integration-${{ inputs.task }}.txt
flags: integration,${{ inputs.task }}-${{ inputs.shard_index }},filter-${{ !inputs.disable_pushdown }}
flags: integration,${{ inputs.task }},${{ inputs.task }}-${{ inputs.shard_index }}
disable_file_fixes: true
fail_ci_if_error: true
verbose: true
- name: Upload coverage information to coveralls
if: always() && inputs.coveralls
if: always()
uses: coverallsapp/github-action@v2
with:
file: ./integration/integration-${{ inputs.task }}.txt
flag-name: integration-${{ inputs.task }}-${{ inputs.shard_index }}-filter-${{ !inputs.disable_pushdown }}
flag-name: integration-${{ inputs.task }}-${{ inputs.shard_index }}
parallel: true
- name: Convert test results to JUnit XML format
if: always() && !inputs.skip_flaky_check
run: >
./bin/go-junit-report -parser=gojson
-in ./integration/*.json.tmp
-out ./integration/${{ inputs.task }}.xml
- name: Upload test results to Codecov
if: always()
uses: codecov/test-results-action@v1
with:
flags: integration,${{ inputs.task }},${{ inputs.task }}-${{ inputs.shard_index }}
file: ./integration/${{ inputs.task }}.xml
# we don't want them on CI
- name: Clean test and fuzz caches
if: always()
run: go clean -testcache -fuzzcache
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
+4 -1
View File
@@ -18,9 +18,12 @@ jobs:
name: Cleanup
runs-on: server
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
+1 -1
View File
@@ -63,7 +63,7 @@ env:
jobs:
conform-pr:
name: Conform PR
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 5
# No `trust` label check because we don't checkout PR's code.
+95
View File
@@ -0,0 +1,95 @@
---
name: CTS
on:
pull_request:
types:
- unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild
- opened
- reopened
- synchronize
push:
branches:
- main
- main-*
- releases/*
env:
GOPATH: /home/runner/go
GOCACHE: /home/runner/go/cache
GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint
GOMODCACHE: /home/runner/go/mod
GOPROXY: https://proxy.golang.org
GOTOOLCHAIN: local
jobs:
run:
name: Run CTS tests
runs-on: ubuntu-24.04
timeout-minutes: 10
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
cache-key: cts
- name: Install Task
run: go generate -x
working-directory: tools
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Start environment
run: bin/task env-up-detach
- name: Run init
run: bin/task init
- name: Build bin/ferretdb
run: bin/task build-host
- name: Wait for and setup environment
run: bin/task env-setup
- name: Start FerretDB (no auth) in the background
run: >
bin/ferretdb
--listen-addr=:27017
--proxy-addr=127.0.0.1:37017
--no-auth
--mode=diff-normal
--postgresql-url='postgres://username:password@127.0.0.1:5432/postgres'
2> ferretdb.log &
- name: Run CTS
if: always()
run: bin/task cts
# we don't want them on CI
- name: Clean test and fuzz caches
if: always()
run: go clean -testcache -fuzzcache
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
+8 -3
View File
@@ -28,22 +28,27 @@ concurrency:
jobs:
contexts:
name: Contexts
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 5
# comment out to enable this job
if: false
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
- name: Debug git status
run: git status
run: |
git status --untracked-files --ignored
git status
- name: Dump github context
run: echo "$GITHUB_CONTEXT"
+11 -7
View File
@@ -10,6 +10,8 @@ on:
push:
branches:
- main
- main-*
- releases/*
schedule:
- cron: "12 0 * * *"
@@ -30,16 +32,20 @@ env:
jobs:
build:
name: Build docs
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: true
# This step will fail with "Error: Resource not accessible by integration"
@@ -52,6 +58,7 @@ jobs:
filesizelimit: 102400b
labelName: use-git-lfs
exclusionPatterns: |
**/*.go
**/*.js
**/*.md
**/go.sum
@@ -68,9 +75,6 @@ jobs:
- name: Build tools
run: bin/task env-pull
- name: Create directory
run: mkdir -p tmp/githubcache
- name: Build docs
run: bin/task docs
env:
@@ -86,7 +90,7 @@ jobs:
# it has to be exactly "docs.tar" for actions/deploy-pages below
- name: Upload archive artifact
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v5
with:
name: docs
path: tmp/docs.tar
@@ -100,7 +104,7 @@ jobs:
github:
name: Deploy to GitHub Pages
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
# exclude pull_request and schedule
+7 -2
View File
@@ -31,15 +31,18 @@ env:
jobs:
format:
name: Format files
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
@@ -60,6 +63,8 @@ jobs:
run: bin/task fmt-yaml
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
-62
View File
@@ -1,62 +0,0 @@
---
# Integration tests for extra configuration.
name: Go Extra
on:
schedule:
- cron: "12 1 * * *" # after Go workflow to reuse "Setup Go" cache
env:
GOPATH: /home/runner/go
GOCACHE: /home/runner/go/cache
GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint
GOMODCACHE: /home/runner/go/mod
GOPROXY: https://proxy.golang.org
GOTOOLCHAIN: local
jobs:
integration:
# job name must be unique; make it unique and nice
name: >
${{ matrix.task }} ${{ matrix.postgres_dockerfile }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}
(pushdown=${{ matrix.disable_pushdown }})
# To avoid conflict with go.yml.
concurrency:
group: ${{ github.workflow }}-integration-${{ matrix.task }}-${{ matrix.postgres_dockerfile }}-${{ matrix.shard_index }}-${{ matrix.disable_pushdown }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
strategy:
fail-fast: false
matrix:
task: [postgresql, sqlite, mysql]
shard_index: [1, 2, 3]
shard_total: [3]
disable_pushdown: [false, true]
include:
- task: postgresql
shard_index: 1
shard_total: 3
disable_pushdown: false
postgres_dockerfile: citus
- task: postgresql
shard_index: 2
shard_total: 3
disable_pushdown: false
postgres_dockerfile: citus
- task: postgresql
shard_index: 3
shard_total: 3
disable_pushdown: false
postgres_dockerfile: citus
# Do not submit to coveralls because it can't handle parallel workflows:
# https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515
uses: ./.github/workflows/_integration.yml
with:
task: ${{ matrix.task }}
shard_index: ${{ matrix.shard_index }}
shard_total: ${{ matrix.shard_total }}
disable_pushdown: ${{ matrix.disable_pushdown }}
postgres_dockerfile: ${{ matrix.postgres_dockerfile }}
coveralls: false
-64
View File
@@ -1,64 +0,0 @@
---
# Integration tests that need access to secrets.
name: Go Trusted
on:
pull_request_target:
types:
# not for "labeled" to prevent two builds for "labeled" and "unlabeled" when labels are changed
- unlabeled # if GitHub Actions stuck, add and remove "not ready" label to force rebuild
- opened
- reopened
- synchronize
push:
branches:
- main
- main-*
- releases/*
schedule:
- cron: "12 1 * * *" # after Go workflow to reuse "Setup Go" cache
env:
GOPATH: /home/runner/go
GOCACHE: /home/runner/go/cache
GOLANGCI_LINT_CACHE: /home/runner/go/cache/lint
GOMODCACHE: /home/runner/go/mod
GOPROXY: https://proxy.golang.org
GOTOOLCHAIN: local
jobs:
integration:
# job name must be unique; make it unique and nice
name: ${{ matrix.task }} ${{ matrix.shard_index }}/${{ matrix.shard_total }}
# Do not run this job in parallel for any PR change or branch push
# to save some resources.
concurrency:
group: ${{ github.workflow }}-integration-${{ matrix.task }}-${{ matrix.shard_index }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
if: >
github.event_name != 'pull_request_target' ||
(
contains(github.event.pull_request.labels.*.name, 'trust') &&
!contains(github.event.pull_request.labels.*.name, 'not ready')
)
strategy:
fail-fast: false
matrix:
task: [hana]
shard_index: [1]
shard_total: [1]
# Do not submit to coveralls because it can't handle parallel workflows:
# https://github.com/lemurheavy/coveralls-public/issues/1636#issuecomment-1529460515
uses: ./.github/workflows/_integration.yml
with:
task: ${{ matrix.task }}
shard_index: ${{ matrix.shard_index }}
shard_total: ${{ matrix.shard_total }}
checkout_ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || ''}}
coveralls: false
secrets:
FERRETDB_HANA_URL: ${{ secrets.FERRETDB_HANA_URL }}
+122 -15
View File
@@ -26,7 +26,7 @@ env:
jobs:
short-test:
name: Short test
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 15
# Do not run this job in parallel for any PR change or branch push.
@@ -36,14 +36,24 @@ jobs:
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
# for branch.txt on pull_request; the correct branch is already checked out on push / schedule
- name: Name branch
if: github.event_name == 'pull_request'
env:
BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
run: git checkout -b $BRANCH
- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
@@ -68,13 +78,15 @@ jobs:
run: go clean -testcache -fuzzcache
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
test:
name: Test
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 15
# Do not run this job in parallel for any PR change or branch push.
@@ -84,14 +96,24 @@ jobs:
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
# for branch.txt on pull_request; the correct branch is already checked out on push / schedule
- name: Name branch
if: github.event_name == 'pull_request'
env:
BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
run: git checkout -b $BRANCH
- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
@@ -101,21 +123,45 @@ jobs:
run: go generate -x
working-directory: tools
# install.sh starts ollama service, stop it since we want to run with `bin/task ollama` with environment variables
- name: Setup Ollama
run: |
curl -fsSL https://ollama.com/install.sh | sudo -E sh
sudo systemctl stop ollama
- name: Start Ollama in the background
run: |
bin/task ollama &
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Start environment
run: bin/task env-up-detach
- name: Pull and load Ollama model
run: |
bin/task ollama-model
- name: Run init
run: bin/task init
- name: Wait for and setup environment
run: bin/task env-setup
- name: Regenerate and reformat
run: bin/task gen
env:
# must be a SUPERUSER to ensure parameter_default is set
DOCUMENTDB_GEN_URL: postgres://username:password@127.0.0.1:5432/postgres
- name: Build bin/ferretdb
run: bin/task build-host
- name: Wait for and setup environment
run: bin/task env-setup
- name: Run unit tests
run: bin/task test-unit TEST_TIMEOUT=10m
env:
@@ -133,19 +179,36 @@ jobs:
verbose: true
- name: Upload coverage information to coveralls
if: always()
uses: coverallsapp/github-action@v2
with:
file: ./cover.txt
flag-name: unit
parallel: true
- name: Convert test results to JUnit XML format
if: always()
run: >
./bin/go-junit-report
-in ./unit.txt
-out ./unit.xml
- name: Upload test results to Codecov
if: always()
uses: codecov/test-results-action@v1
with:
flags: unit
file: ./unit.xml
# we don't want them on CI
- name: Clean test and fuzz caches
if: always()
run: go clean -testcache -fuzzcache
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
@@ -164,36 +227,59 @@ jobs:
strategy:
fail-fast: false
matrix:
task: [postgresql, sqlite]
task: [postgresql, yugabytedb]
shard_index: [1, 2, 3, 4, 5]
shard_total: [5]
include:
- { task: "mongodb", shard_index: 1, shard_total: 1 }
- { task: "yugabytedb", skip_flaky_check: true } # Too many failures for now
uses: ./.github/workflows/_integration.yml
with:
task: ${{ matrix.task }}
shard_index: ${{ matrix.shard_index }}
shard_total: ${{ matrix.shard_total }}
coveralls: true
skip_flaky_check: ${{ matrix.skip_flaky_check || false }}
integration-no-xfail:
name: postgresql no-xfail
# Do not run this job in parallel for any PR change or branch push
# to save some resources.
concurrency:
group: ${{ github.workflow }}-integration-no-xfail-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
# disable for now
if: false
# if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
uses: ./.github/workflows/_integration.yml
with:
task: postgresql
shard_index: 1
shard_total: 1
test_timeout: 5m
no_xfail: true
# https://github.com/lemurheavy/coveralls-public/issues/1636
submit-coveralls:
name: Submit final coveralls report
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
needs: [test, integration]
if: always() && (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready'))
steps:
- name: Submit coveralls
if: always()
uses: coverallsapp/github-action@v2
with:
parallel-finished: true
env-data:
name: env-data
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
# Do not run this job in parallel for any PR change or branch push.
@@ -203,10 +289,13 @@ jobs:
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
@@ -220,6 +309,13 @@ jobs:
run: go generate -x
working-directory: tools
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Start environment
run: bin/task env-up-detach
@@ -242,7 +338,7 @@ jobs:
golangci-lint:
name: golangci-lint
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
# Do not run this job in parallel for any PR change or branch push.
@@ -252,14 +348,25 @@ jobs:
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
issues: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `golangci-lint run --new` to work
lfs: false # LFS is used only by website
# for branch.txt on pull_request; the correct branch is already checked out on push / schedule
- name: Name branch
if: github.event_name == 'pull_request'
env:
BRANCH: ${{ github.head_ref }} # see https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
run: git checkout -b $BRANCH
- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
+16 -10
View File
@@ -26,7 +26,7 @@ on:
tags:
- "*"
schedule:
- cron: "10 1 * * 1" # after cleanup
- cron: "10 1 * * 1" # after cleanup and DocumentDB
# Do not run this workflow in parallel for any PR change or branch/tag push
# to prevent concurrent pushes for the same Docker image tag
@@ -64,7 +64,7 @@ jobs:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
if: github.event_name != 'pull_request_target'
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
@@ -72,7 +72,7 @@ jobs:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout pull request code
if: github.event_name == 'pull_request_target'
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
@@ -136,9 +136,13 @@ jobs:
username: ferretdbbot
password: ${{ secrets.QUAY_TOKEN }}
- name: Build and push all-in-one Docker images ${{ steps.tag.outputs.all_in_one_images }}
if: steps.tag.outputs.all_in_one_images != ''
run: bin/task docker-all-in-one-push DOCKER_IMAGES=${{ steps.tag.outputs.all_in_one_images }}
- name: Build and push evaluation development Docker images ${{ steps.tag.outputs.eval_dev_images }}
if: steps.tag.outputs.eval_dev_images != ''
run: bin/task docker-eval-dev-push DOCKER_IMAGES=${{ steps.tag.outputs.eval_dev_images }}
- name: Build and push evaluation Docker images ${{ steps.tag.outputs.eval_images }}
if: steps.tag.outputs.eval_images != ''
run: bin/task docker-eval-push DOCKER_IMAGES=${{ steps.tag.outputs.eval_images }}
- name: Build and push development Docker images ${{ steps.tag.outputs.development_images }}
if: steps.tag.outputs.development_images != ''
@@ -152,7 +156,7 @@ jobs:
run: bin/task build-development
- name: Upload development binaries
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v5
with:
name: bin-dev
path: tmp/bin-dev/
@@ -163,7 +167,7 @@ jobs:
run: bin/task build-production
- name: Upload production binaries
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v5
with:
name: bin
path: tmp/bin/
@@ -174,7 +178,7 @@ jobs:
run: bin/task packages
- name: Upload .deb packages
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v5
with:
name: debs
path: tmp/debs/
@@ -182,7 +186,7 @@ jobs:
if-no-files-found: error
- name: Upload .rpm packages
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v5
with:
name: rpms
path: tmp/rpms/
@@ -190,6 +194,8 @@ jobs:
if-no-files-found: error
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
+5 -2
View File
@@ -31,15 +31,18 @@ env:
jobs:
security-scan:
name: Security scan
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
@@ -1,5 +1,5 @@
---
name: JS
name: Test scripts
on:
pull_request:
types:
@@ -9,6 +9,7 @@ on:
- synchronize
paths:
- build/legacy-mongo-shell/*.js
- build/testscripts/*.sql
push:
branches:
- main
@@ -31,17 +32,20 @@ env:
GOTOOLCHAIN: local
jobs:
testjs:
name: Test script
runs-on: ubuntu-22.04
run:
name: Run test scripts
runs-on: ubuntu-24.04
timeout-minutes: 10
if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'not ready')
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for `git describe` to work
lfs: false # LFS is used only by website
@@ -49,12 +53,19 @@ jobs:
- name: Setup Go
uses: FerretDB/github-actions/setup-go@main
with:
cache-key: testjs
cache-key: testscripts
- name: Install Task
run: go generate -x
working-directory: tools
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Start environment
run: bin/task env-up-detach
@@ -67,20 +78,26 @@ jobs:
- name: Wait for and setup environment
run: bin/task env-setup
- name: Start FerretDB in the background
- name: Start FerretDB (no auth) in the background
run: >
bin/ferretdb
--listen-addr=:27017
--proxy-addr=127.0.0.1:47017
--proxy-addr=127.0.0.1:37017
--no-auth
--mode=diff-normal
--handler=pg
--postgresql-url='postgres://username@127.0.0.1:5432/ferretdb'
--postgresql-url='postgres://username:password@127.0.0.1:5432/postgres'
2> ferretdb.log &
- name: Run testjs for MongoDB
run: bin/task testjs TESTJS_PORT=47017
- name: Run testsql for PostgreSQL
if: always()
run: bin/task testsql
- name: Run testjs for FerretDB
- name: Run testjs for MongoDB (no auth)
if: always()
run: bin/task testjs TESTJS_PORT=37017
- name: Run testjs for FerretDB (no auth)
if: always()
run: bin/task testjs
- name: Compress FerretDB log before upload
@@ -89,7 +106,7 @@ jobs:
- name: Upload FerretDB log
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v5
with:
name: ferretdb-log
path: ferretdb-log.zip
@@ -99,6 +116,7 @@ jobs:
- name: Show FerretDB log
if: always()
run: |
echo "::remove-matcher owner=go::"
cat ferretdb.log
rm ferretdb.log
@@ -108,6 +126,8 @@ jobs:
run: go clean -testcache -fuzzcache
- name: Check dirty
if: always()
run: |
git status --untracked-files --ignored
git status
git diff --exit-code
+10 -7
View File
@@ -26,7 +26,7 @@ env:
jobs:
blog-preview:
name: Preview blog
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
if: >
@@ -39,10 +39,13 @@ jobs:
name: cloudflare-dev-blog
url: ${{ steps.extract.outputs.extracted_url }}
permissions:
contents: read
steps:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for wrangler to work
lfs: true
@@ -68,9 +71,6 @@ jobs:
- name: Build tools
run: bin/task env-pull
- name: Create directory
run: mkdir -p tmp/githubcache
- name: Build blog
run: bin/task blog
env:
@@ -93,7 +93,7 @@ jobs:
docs-preview:
name: Preview docs
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
timeout-minutes: 10
if: >
@@ -105,6 +105,9 @@ jobs:
name: cloudflare-dev-docs
url: ${{ steps.extract.outputs.extracted_url }}
permissions:
contents: read
steps:
# We can't easily download build results from the Docs workflow:
# https://github.com/actions/download-artifact/issues/172
@@ -113,7 +116,7 @@ jobs:
# TODO https://github.com/FerretDB/github-actions/issues/211
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@v5
with:
fetch-depth: 0 # for wrangler to work
lfs: true
+9 -4
View File
@@ -1,14 +1,19 @@
# Please use global .gitignore file for .vscode/, .idea/, go.work, .envrc, etc.
# This file is used only for repository-specific patterns.
# Please use global .gitignore file for .DS_Store, .idea/, go.work, .envrc, etc.
# See https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files#configuring-ignored-files-for-all-repositories-on-your-computer
docker-compose.override.yml
state.json
telemetry.json
*.bc
*.o
*.so
*.out
*.sqlite*
*.tmp
*.txt
!website/static/*.txt
*.xml
.task/
bin/
+4
View File
@@ -0,0 +1,4 @@
[submodule "documentdb"]
path = build/postgres-documentdb/documentdb
url = https://github.com/FerretDB/documentdb.git
branch = ferretdb
+9 -15
View File
@@ -1,10 +1,11 @@
---
# New or experimental linters that should pass (or be reconfigured)
# for new FerretDB and tools code (compared to `origin/main-v1`).
# for new FerretDB and tools code (compared to `origin/main`).
run:
timeout: 3m
# bin/golangci-lint linters -c .golangci-new.yml
linters-settings:
asasalint:
exclude:
@@ -18,8 +19,7 @@ linters-settings:
check-blank: false
disable-default-exclusions: false
exclude-functions:
- (*database/sql.Rows).Close
- (*github.com/FerretDB/FerretDB/internal/util/fsql.Rows).Close
- (*github.com/jackc/pgx/v5.Conn).Close
errorlint:
# see caveats at https://github.com/polyfloyd/go-errorlint#fmterrorf-wrapping-verb
errorf: false
@@ -41,14 +41,6 @@ linters-settings:
- name: package-comments
staticcheck:
checks: ["all"]
tagalign:
sort: true
order:
- name
- cmd
- arg
- default
- help
wsl:
# https://github.com/bombsimon/wsl/blob/master/doc/configuration.md
strict-append: false
@@ -70,6 +62,7 @@ linters:
- asciicheck
- depguard
- exhaustive
- exhaustruct
- forbidigo
- gci
- gochecksumtype
@@ -86,6 +79,8 @@ linters:
- misspell
- nolintlint
- sloglint
- tagalign
- unparam
- unused
- whitespace
@@ -104,8 +99,6 @@ linters:
- err113
- errchkjson
- errname
- exhaustruct
- exportloopref
- fatcontext
- forcetypeassert
- funlen
@@ -123,6 +116,7 @@ linters:
- gosec
- gosmopolitan
- grouper
- iface
- inamedparam
- interfacebloat
- intrange
@@ -147,6 +141,7 @@ linters:
- promlinter
- protogetter
- reassign
- recvcheck
- rowserrcheck
- sloglint
- spancheck
@@ -160,7 +155,6 @@ linters:
- tparallel
- typecheck
- unconvert
- unparam
- usestdlibvars
- varnamelen
- wastedassign
@@ -174,7 +168,7 @@ issues:
max-issues-per-linter: 0
max-same-issues: 0
new-from-rev: origin/main-v1
new-from-rev: origin/main
# whole-files: true
exclude-use-default: false
+90 -61
View File
@@ -16,8 +16,9 @@ linters-settings:
desc: use `context` package instead
- pkg: golang.org/x/exp/slices
desc: use `slices` package instead
- pkg: maps
desc: use `golang.org/x/exp/maps` package instead
- pkg: golang.org/x/exp/maps
desc: use `maps` package instead
- pkg: github.com/jackc/pgconn
desc: use `github.com/jackc/pgx/v5/pgconn` package instead
- pkg: github.com/jackc/pgproto3
@@ -29,87 +30,91 @@ linters-settings:
- pkg: github.com/jackc/pgx/v4
desc: use `github.com/jackc/pgx/v5` package instead
# TODO https://github.com/FerretDB/FerretDB/issues/4798
# - pkg: go.mongodb.org/mongo-driver/bson
# desc: use `go.mongodb.org/mongo-driver/v2/bson` package instead
# - pkg: go.mongodb.org/mongo-driver/mongo
# desc: use `go.mongodb.org/mongo-driver/v2/mongo` package instead
# - pkg: github.com/xdg-go/scram
# desc: use `github.com/FerretDB/FerretDB/v2/internal/util/scram` package instead
ferretdb-in-tools:
files:
- "**/tools/**/*.go"
deny:
- pkg: github.com/FerretDB/FerretDB/internal/
- pkg: github.com/FerretDB/FerretDB/v2/internal/
desc: tools should not import FerretDB code
types:
files:
- "**/internal/util/logging/*.go"
deny:
- pkg: github.com/FerretDB/FerretDB/internal/types
sjson:
files:
- $all
- "!**/internal/backends/sqlite/*.go"
- "!**/internal/backends/postgresql/*.go"
- "!**/internal/backends/postgresql/metadata/*.go"
- "!**/internal/backends/mysql/*.go"
- "!**/internal/backends/mysql/metadata/*.go"
- "!**/internal/backends/hana/*.go"
deny:
- pkg: github.com/FerretDB/FerretDB/internal/handler/sjson
common:
files:
- "**/internal/backends/**.go"
- "**/internal/clientconn/cursor/*.go"
deny:
- pkg: github.com/FerretDB/FerretDB/internal/handler/common
handlererrors:
wirebson:
files:
- $all
- "!**/cmd/envtool/*.go"
- "!**/internal/clientconn/*.go"
- "!**/internal/dataapi/server/*.go"
- "!**/internal/documentdb/**.go"
- "!**/internal/driver/*.go"
- "!**/internal/handler/**.go"
- "!**/internal/mcp/*.go"
- "!**/internal/mongoerrors/*.go"
- "!**/internal/util/logging/*.go"
- "!**/internal/util/scram/*.go"
- "!**/internal/util/testutil/*.go"
deny:
- pkg: github.com/FerretDB/FerretDB/internal/handler/handlererrors
backends:
- pkg: github.com/FerretDB/wire/wirebson
documentdb:
files:
- "**/internal/clientconn/cursor/*.go"
- "**/internal/documentdb/genwrap/*.go"
deny:
- pkg: github.com/FerretDB/FerretDB/internal/backends
- pkg: github.com/FerretDB/FerretDB/v2/internal/documentdb
cursor:
files:
- $all
- "!**/internal/documentdb/**.go"
deny:
- pkg: github.com/FerretDB/FerretDB/v2/internal/documentdb/cursor
exhaustive:
default-signifies-exhaustive: false
exhaustruct:
include:
- TODO
forbidigo:
forbid:
- p: ^\Qpgxpool.Pool.AcquireFunc\E$
pkg: ^\Qgithub.com/jackc/pgx/v5/pgxpool\E$
msg: Use `Acquire` with `defer Release()` instead.
# integration tests use a different configuration file
# TODO https://github.com/FerretDB/FerretDB/issues/4798
- p: ^\Qbson.E\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson\E$
msg: Use `*types.Document` or `*types.Array` instead.
- p: ^\Qprimitive.E\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson/primitive\E$
msg: Use `*types.Document` or `*types.Array` instead.
msg: Use `*wirebson.Document` or `*wirebson.Array` instead.
- p: ^\Qbson.D\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson\E$
msg: Use `*types.Document` instead.
- p: ^\Qprimitive.D\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson/primitive\E$
msg: Use `*types.Document` instead.
msg: Use `*wirebson.Document` instead.
- p: ^\Qbson.M\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson\E$
msg: Use `*types.Document` instead.
- p: ^\Qprimitive.M\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson/primitive\E$
msg: Use `*types.Document` instead.
msg: Use `*wirebson.Document` instead.
- p: ^\Qbson.A\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson\E$
msg: Use `*types.Array` instead.
- p: ^\Qprimitive.A\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/bson/primitive\E$
msg: Use `*types.Array` instead.
msg: Use `*wirebson.Array` instead.
- p: ^\Qbson.E\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/v2/bson\E$
msg: Use `*wirebson.Document` or `*wirebson.Array` instead.
- p: ^\Qbson.D\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/v2/bson\E$
msg: Use `*wirebson.Document` instead.
- p: ^\Qbson.M\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/v2/bson\E$
msg: Use `*wirebson.Document` instead.
- p: ^\Qbson.A\E$
pkg: ^\Qgo.mongodb.org/mongo-driver/v2/bson\E$
msg: Use `*wirebson.Array` instead.
exclude-godoc-examples: true
analyze-types: true
gci:
sections:
- standard
- default
- prefix(github.com/FerretDB/FerretDB)
- prefix(github.com/FerretDB/FerretDB/v2/)
goconst:
min-occurrences: 5
ignore-tests: true
@@ -123,6 +128,8 @@ linters-settings:
- BUG
- FIXME
- HACK
- lala
- lalala
goheader:
# Please do not update it, including bumping a year.
template: |-
@@ -140,10 +147,12 @@ linters-settings:
See the License for the specific language governing permissions and
limitations under the License.
goimports:
local-prefixes: github.com/FerretDB/FerretDB
local-prefixes: github.com/FerretDB/FerretDB/v2
gomodguard:
blocked:
modules: []
modules:
- github.com/FerretDB/FerretDB:
reason: "Use `github.com/FerretDB/FerretDB/v2/...` instead."
gosimple:
checks: ["all"]
govet:
@@ -153,15 +162,14 @@ linters-settings:
- nilness
- shadow
importas:
no-unaliased: false
no-extra-aliases: true
# https://github.com/julz/importas/issues/15
no-unaliased: true
no-extra-aliases: false
alias:
- pkg: runtime/debug
alias: runtimedebug
- pkg: github.com/prometheus/client_model/go
alias: dto
- pkg: modernc.org/sqlite
alias: sqlite3
- pkg: modernc.org/sqlite/lib
alias: sqlite3lib
- pkg: go.opentelemetry.io/otel/attribute
alias: otelattribute
- pkg: go.opentelemetry.io/otel/codes
@@ -172,10 +180,8 @@ linters-settings:
alias: otelsdkresource
- pkg: go.opentelemetry.io/otel/sdk/trace
alias: otelsdktrace
- pkg: go.opentelemetry.io/otel/semconv/v1.21.0
- pkg: go.opentelemetry.io/otel/semconv/v1.34.0
alias: otelsemconv
- pkg: github.com/FerretDB/FerretDB/internal/backends/mysql/metadata/pool
alias: mysqlpool
# ineffassign
lll:
line-length: 130
@@ -211,7 +217,23 @@ linters-settings:
checks:
- all
- -SA1019 # ignore deprecation errors in existing code; new code is checked by the other configuration
# unused
tagalign:
sort: true
order:
- name
- cmd
- arg
- default
- help
unparam:
check-exported: false
unused:
field-writes-are-uses: false
post-statements-are-reads: false
exported-fields-are-used: false
parameters-are-used: true # it should be false, TODO https://github.com/FerretDB/FerretDB-DocumentDB/issues/823
local-variables-are-used: false
generated-is-used: false
whitespace:
multi-if: false
multi-func: false
@@ -222,6 +244,7 @@ linters:
- asciicheck
- depguard
- exhaustive
- exhaustruct
- forbidigo
- gci
- gochecksumtype
@@ -241,6 +264,8 @@ linters:
- revive
- sloglint
- staticcheck
- tagalign
- unparam
- unused
- whitespace
@@ -253,3 +278,7 @@ issues:
# we don't want to introduce that constant
- linters: [goconst]
text: "^string `_id` has \\d+ occurrences, make it a constant"
- linters: [forbidigo]
path: "\\Qferretdb/ferretdb_test.go"
text: "\\Quse of `bson.D` forbidden"
+3
View File
@@ -1,3 +1,6 @@
# We can't have it in build/.prettierignore due to https://github.com/prettier/prettier/issues/15428
# Textlint uses the same format, so we re-use the file.
build/postgres-documentdb/documentdb/**
internal/dataapi/api/openapi.json
internal/dataapi/api/openapi-original.json
website/static/codapi/snippet.js
+573 -1717
View File
File diff suppressed because it is too large. Load diff
+65 -47
View File
@@ -30,7 +30,7 @@ On macOS and Windows, [Docker Desktop](https://www.docker.com/products/docker-de
On Windows, it should be [configured to use WSL 2](https://docs.docker.com/desktop/windows/wsl/) without any distro;
all commands should be run on the host.
You will need Go 1.24 or later on the host.
You will need Go 1.25 or later on the host.
If your package manager doesn't provide it yet,
please install it from [go.dev](https://go.dev/dl/).
@@ -106,8 +106,6 @@ To help us accurately identify the cause, we encourage you to include a pull req
Please write the test script in [build/legacy-mongo-shell/test.js](build/legacy-mongo-shell/test.js).
You can find an overview of the available assertions [here](build/legacy-mongo-shell/README.md).
Use these assertions to validate your test's assumptions and invariants.
You can also find an example of how to prepare a test script in
[build/legacy-mongo-shell/test.example.js](build/legacy-mongo-shell/test.example.js).
With `task` installed (see above), you may test your script using following steps:
@@ -125,12 +123,20 @@ With `task` installed (see above), you may do the following:
1. Start the development environment with `task env-up`.
2. Run all tests with `task test` (see [below](#running-tests)).
3. Start FerretDB with `task run`, `task run-sqlite` or `task run-proxy`.
(See [Operation modes](https://docs.ferretdb.io/v1.24/configuration/operation-modes/) page in our documentation.)
3. Start FerretDB with `task run`.
The development environment uses `diff-normal` mode as the default mode.
Set preferred mode such as `task run MODE=diff-proxy`.
(See [Operation modes](https://docs.ferretdb.io/configuration/operation-modes/) page in our documentation.)
4. Fill collections in the `test` database with data for experiments with `task env-data`.
5. Run `mongosh` with `task mongosh`.
See what collections were created by the previous command with `show collections`.
#### Commands for authenticated connection
1. Start the development environment with `task env-up`.
2. Start FerretDB with `task run-secure MODE=normal` or `task run-secure MODE=proxy`.
3. Run `task mongosh-secure` to establish an authenticated connection.
### Code overview
The directory `cmd` provides commands implementation.
@@ -141,19 +147,19 @@ They are installed into `bin/` by `cd tools; go generate -x`.
The `internal` subpackages contain most of the FerretDB code:
- `types` package provides Go types matching BSON types that don't have built-in Go equivalents:
we use `int32` for BSON's int32, but `types.ObjectID` for BSON's ObjectId.
- `bson` package provides converters from/to BSON for built-in and `types` types.
- `wire` package provides wire protocol implementation.
- `clientconn` package provides client connection implementation.
It accepts client connections, reads `wire`/`bson` protocol messages, and passes them to `handler`.
Responses are then converted to `wire`/`bson` messages and sent back to the client.
- `handler` contains implementations of command handlers.
They use `types` and `wire` packages, but `bson` package details are hidden.
- `handler/sjson` provides converters from/to SJSON for built-in and `types` types.
SJSON adds some extensions to JSON for keeping object keys in order,
preserving BSON type information in the values themselves, etc.
It is used by some backends.
- `handler` provides implementations of command handlers.
The logic of commands like `find` and `aggregate` is there.
- `dataapi` provides an Data API wrapper for `handler`.
It allows FerretDB to be used over HTTP instead of MongoDB wire protocol.
- `clientconn` provides wire protocol server implementation.
It accepts client connections, reads wire protocol messages, and passes them to `handler`.
Responses are then converted and sent back to the client.
- `documentdb` provides DocumentDB extension integration.
- `mongoerrors` provides MongoDB-compatible error types and codes.
They are used by `handler` and `documentdb`.
The wire protocol implementation and BSON handling code were extracted into a separate repository:
https://github.com/FerretDB/wire
#### Running tests
@@ -169,13 +175,13 @@ They use the Go MongoDB driver like a regular user application.
They could test any MongoDB-compatible database (such as FerretDB or MongoDB itself) via a regular TCP or TLS port
or Unix domain socket.
They also could test in-process FerretDB instances
(meaning that integration tests start and stop them themselves) with a given backend.
(meaning that integration tests start and stop them themselves).
Finally, some integration tests (so-called compatibility or "compat" tests) connect to two systems
("target" for FerretDB and "compat" for MongoDB) at the same time,
send the same queries to both, and compare results.
You can run them with:
- `task test-integration-postgresql` for in-process FerretDB with PostgreSQL backend and MongoDB;
- `task test-integration-postgresql` for in-process FerretDB and MongoDB;
- `task test-integration-mongodb` for MongoDB only, skipping compat tests;
- or `task test-integration` to run all in parallel.
@@ -185,7 +191,7 @@ If tests fail and the output is too confusing, try running them sequentially by
You can also run `task -C 1` to limit the number of concurrent tasks, which is useful for debugging.
To run a subset of integration tests and test cases, you may use Task variable `TEST_RUN`.
For example, to run all tests related to the `getMore` command implementation for in-process FerretDB with PostgreSQL backend
For example, to run all tests for the `getMore` command with in-process FerretDB
you may use `task test-integration-postgresql TEST_RUN='(?i)GetMore'`.
Finally, since all tests just run `go test` with various arguments and flags under the hood
@@ -194,12 +200,10 @@ you may also use all standard `go` tool facilities,
including [`GOFLAGS` environment variable](https://pkg.go.dev/cmd/go#hdr-Environment_variables).
For example:
- to run all tests related to the `getMore` command implementation in-process FerretDB with PostgreSQL backend
with all subtests running sequentially,
you may use `env GOFLAGS='-parallel=1' task test-integration-postgresql TEST_RUN='(?i)GetMore'`;
- to run all tests for in-process FerretDB with SQLite backend
with [Go execution tracer](https://pkg.go.dev/runtime/trace) enabled,
you may use `env GOFLAGS='-trace=trace.out' task test-integration-sqlite`.
- to run all tests for the `getMore` command sequentially,
you may use `env GOFLAGS='-parallel=1 -p=1' task test-integration-postgresql TEST_RUN='(?i)GetMore'`;
- to run all tests with MongoDB with [Go execution tracer](https://pkg.go.dev/runtime/trace) enabled
you may use `env GOFLAGS='-trace=trace.out' task test-integration-mongodb`.
<!-- textlint-disable one-sentence-per-line -->
@@ -239,10 +243,10 @@ and some other pages such as [Spelling](https://go.dev/wiki/Spelling).
Some of our idiosyncrasies are documented below.
1. We use type switches over BSON types in many places in our code.
The order of `case`s follows this order: https://pkg.go.dev/github.com/FerretDB/FerretDB/internal/types#hdr-Mapping
The order of `case`s follows this order: https://pkg.go.dev/github.com/FerretDB/wire/wirebson#hdr-Types
It may seem random, but it is only pseudo-random and follows BSON spec: https://bsonspec.org/spec.html
2. We generally pass and return `struct`s by pointers.
There are some exceptions like `types.Path` that have value semantics, but when in doubt – use pointers.
There are some exceptions like `time.Time` that have value semantics, but when in doubt – use pointers.
#### Comments conventions
@@ -271,21 +275,23 @@ Some of our idiosyncrasies are documented below.
#### Integration tests conventions
We prefer our integration tests to be straightforward,
branchless (with a few, if any, `if` and `switch` statements),
and backend-independent.
Ideally, the same test should work for both FerretDB with all backends and MongoDB.
We prefer our integration tests to be straightforward and
branchless (with a few, if any, `if` and `switch` statements).
Ideally, the same test should work for both FerretDB and MongoDB.
If that's impossible without some branching, use helpers exported from the `setup` package,
such us `FailsForFerretDB`, `SkipForMongoDB`, etc.
Please use expected failure rather than skipping the test, whenever it's possible.
It makes tracking development progress much easier.
The bar for using other ways of branching, such as checking error codes and messages, is very high.
Writing separate tests might be much better than making a single test that checks error text.
Also, we should use driver methods as much as possible instead of testing commands directly via `RunCommand`.
We have a lot of existing helpers to convert the driver's `bson.D` values to our `*types.Document` values,
to compare them, etc.
In most cases, they should be used instead of (deprecated) `bson.D.Map()`,
`bson.Unmarshal`, decoding into a struct with `bson` tags, comparing fields one-by-one, etc.
In most cases, helpers such as `AssertEqualDocuments` are used to compare `bson.D` documents.
The differences in field order, types, missing fields and others should be reported.
If differences are expected such as `version` field, only the field key can be used for `bson.D` document comparison.
Use of (deprecated) `bson.D.Map()` or converting `bson.D` to other types using helpers should be avoided when possible.
The bar for adding new helpers is very high.
Please check all existing ones.
@@ -297,30 +303,42 @@ For example:
const doubleMaxPrec = float64(1<<53 - 1) // 9007199254740991.0: largest double values that could be represented as integer exactly
```
Avoid writing tests that cannot run in parallel with other tests.
The `integration` directory contains multiple test packages,
and tests from other packages may run at the same time.
Such test becomes flaky and hard to maintain.
#### Integration tests naming guidelines
<!-- https://github.com/FerretDB/FerretDB/issues/694 -->
1. Test names should include the name of the command being tested.
For instance, `TestDistinct` for testing the distinct command.
Use `Test<Command><Operator><Scenario>` format for test names, omitting `<Operator>` and `<Scenario>` if unnecessary.
1. Include the name of the command (and the operator if any).
For example, `TestDistinct` for testing the distinct command.
2. Compatibility tests should have `Compat` in the name, following the command.
For example, `TestDistinctCompat`.
3. If the test doesn't use driver method but runs a command directly via `RunCommand`,
the suffix `Command` should be added.
3. If the test runs a command directly via `RunCommand`, add the suffix `Command`.
For example, `TestDistinctCommand`.
4. If the test is both compat and runs a command, the suffix `CommandCompat` should be added.
4. If the test is both compat and runs a command, add the suffix `CommandCompat`.
For example, `TestInsertCommandCompat`.
5. If the file consists of compatibility tests, add the `_compat` suffix.
For example, `distinct_compat_test.go`.
6. Test names should be descriptive and provide information about the functionality or condition being tested.
If the test is checking for a specific error scenario, include the error scenario in the name.
6. Use descriptive test names to include the condition or the specific error being tested.
For example, `TestAggregateCompatSortDotNotation` for compatibility tests
on `aggregate` command `$sort` operator on a document field.
7. Keep test names concise, avoiding overly cryptic names.
Use abbreviations when appropriate.
8. Avoid including test data in the name to maintain clarity and prevent excessively long names.
9. Test case names should follow `TitleCase` capitalization style.
9. Use `TitleCase` capitalization style for test case names.
No spaces, dashes or underscores should be used neither for test names nor for test case names.
10. Keep the concatenation of test name segments (test, subtests) within 64 characters
10. Use [BSON notation](https://bsonspec.org/spec.html) vocabulary.
For example, use `Double` and not `float64`.
11. Keep the concatenation of test name segments (test, subtests) within 64 characters
to satisfy the maximum limit for database names.
12. Use a package for tests in the same scope.
Separating into packages can reduce binary sizes and may enhance performance.
For example, use `users` package for user management test files.
### Submitting code changes
@@ -389,4 +407,4 @@ Before submitting a pull request, please make sure that:
created from the [current state](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)
of our main branch.
2. Documentation is formatted, linted, and built with `task docs`.
3. Documentation is written according to our [writing guide](https://docs.ferretdb.io/v1.24/contributing/writing-guide/).
3. Documentation is written according to our [writing guide](https://docs.ferretdb.io/contributing/writing-guide/).
-2
View File
@@ -1,2 +0,0 @@
FerretDB database system (https://www.ferretdb.com/).
Copyright 2021 FerretDB Inc.
+39 -47
View File
@@ -1,28 +1,34 @@
# FerretDB
[![Go Reference](https://pkg.go.dev/badge/github.com/FerretDB/FerretDB/ferretdb.svg)](https://pkg.go.dev/github.com/FerretDB/FerretDB/ferretdb)
<!-- textlint-disable one-sentence-per-line -->
> [!TIP]
> Looking for FerretDB v1?
> It is [there](https://github.com/FerretDB/FerretDB/tree/main-v1).
<!-- textlint-enable one-sentence-per-line -->
[![Go Reference](https://pkg.go.dev/badge/github.com/FerretDB/FerretDB/v2/ferretdb.svg)](https://pkg.go.dev/github.com/FerretDB/FerretDB/v2/ferretdb)
[![Go](https://github.com/FerretDB/FerretDB/actions/workflows/go.yml/badge.svg?branch=main)](https://github.com/FerretDB/FerretDB/actions/workflows/go.yml)
[![codecov](https://codecov.io/gh/FerretDB/FerretDB/branch/main-v1/graph/badge.svg?token=JZ56XFT3DM)](https://codecov.io/gh/FerretDB/FerretDB)
[![codecov](https://codecov.io/gh/FerretDB/FerretDB/branch/main/graph/badge.svg?token=JZ56XFT3DM)](https://codecov.io/gh/FerretDB/FerretDB)
[![Security](https://github.com/FerretDB/FerretDB/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/FerretDB/FerretDB/actions/workflows/security.yml)
[![Packages](https://github.com/FerretDB/FerretDB/actions/workflows/packages.yml/badge.svg?branch=main)](https://github.com/FerretDB/FerretDB/actions/workflows/packages.yml)
[![Docs](https://github.com/FerretDB/FerretDB/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/FerretDB/FerretDB/actions/workflows/docs.yml)
FerretDB was founded to become the de-facto open-source substitute to MongoDB.
FerretDB is an open-source proxy, converting the MongoDB 5.0+ wire protocol queries to SQL -
using PostgreSQL or SQLite as a database engine.
FerretDB is an open-source alternative to MongoDB.
It is a proxy that converts MongoDB 5.0+ wire protocol queries to SQL
and uses PostgreSQL with [DocumentDB extension](https://github.com/documentdb/documentdb) as a database engine.
```mermaid
flowchart LR
A["Any application<br>Any MongoDB driver"]
F{{FerretDB}}
P[(PostgreSQL)]
S[("SQLite")]
P[(PostgreSQL<br>DocumentDB extension)]
A -- "MongoDB protocol<br>BSON" --> F
F -- "PostgreSQL protocol<br>SQL" --> P
F -. "SQLite library<br>SQL" .-> S
```
## Why do we need FerretDB?
@@ -31,7 +37,7 @@ MongoDB was originally an eye-opening technology for many of us developers,
empowering us to build applications faster than using relational databases.
In its early days, its ease-to-use and well-documented drivers made MongoDB one of the simplest database solutions available.
However, as time passed, MongoDB abandoned its open-source roots;
changing the license to [SSPL](https://www.mongodb.com/licensing/server-side-public-license) - making it unusable for many open source and early-stage commercial projects.
changing the license to [SSPL](https://www.ferretdb.com/sspl) - making it unusable for many open-source and early-stage commercial projects.
Most MongoDB users do not require any advanced features offered by MongoDB;
however, they need an easy-to-use open-source document database solution.
@@ -45,84 +51,70 @@ Features are constantly being added to further increase compatibility and perfor
We welcome all contributors.
See our [public roadmap](https://github.com/orgs/FerretDB/projects/2/views/1),
a list of [known differences with MongoDB](https://docs.ferretdb.io/v1.24/diff/),
lists of [known differences and supported commands](https://docs.ferretdb.io/migration/compatibility/),
and [contributing guidelines](CONTRIBUTING.md).
## Quickstart
Run this command to start FerretDB with PostgreSQL backend:
Run this command to start FerretDB with PostgreSQL, make sure to update `<username>` and `<password>`:
```sh
docker run -d --rm --name ferretdb -p 27017:27017 ghcr.io/ferretdb/all-in-one
docker run -d --rm --name ferretdb -p 27017:27017 \
-e POSTGRES_USER=<username> \
-e POSTGRES_PASSWORD=<password> \
ghcr.io/ferretdb/ferretdb-eval:2
```
Alternatively, run this command to start FerretDB with SQLite backend:
```sh
docker run -d --rm --name ferretdb -p 27017:27017 -e FERRETDB_HANDLER=sqlite ghcr.io/ferretdb/all-in-one
```
This command will start a container with FerretDB, PostgreSQL/SQLite, and MongoDB Shell for quick testing and experiments.
This command will start a container with FerretDB, pre-packaged PostgreSQL with DocumentDB extension, and MongoDB Shell for quick testing and experiments.
However, it is unsuitable for production use cases because it keeps all data inside and loses it on shutdown.
See our [Docker quickstart guide](https://docs.ferretdb.io/v1.24/quickstart-guide/docker/) for instructions
See our [installation guides](https://docs.ferretdb.io/installation/) for instructions
that don't have those problems.
With that container running, you can:
- Connect to it with any MongoDB client application using MongoDB URI `mongodb://127.0.0.1:27017/`.
- Connect to it using MongoDB Shell by just running `mongosh`.
- Connect to it with any MongoDB client application using the MongoDB URI `mongodb://<username>:<password>@127.0.0.1:27017/`.
- Connect to it using the MongoDB Shell by just running `mongosh`.
If you don't have it installed locally, you can run `docker exec -it ferretdb mongosh`.
- For the PostgreSQL backend, connect to it by running `docker exec -it ferretdb psql -U username ferretdb`.
FerretDB uses PostgreSQL schemas for MongoDB databases.
So, if you created some collections in the `test` database using any MongoDB client,
you can switch to it by running `SET search_path = 'test';` query
and see a list of PostgreSQL tables by running `\d` `psql` command.
- For the SQLite backend, connect to it by running `docker exec -it ferretdb sqlite3 /state/<database>.sqlite`.
So, if you created some collections in the `test` database using any MongoDB client,
run `docker exec -it ferretdb sqlite3 /state/test.sqlite`
and see a list of SQLite tables by running `.tables` command.
- For PostgreSQL, connect to it by running `docker exec -it ferretdb psql -U <username> postgres`.
You can stop the container with `docker stop ferretdb`.
We also provide binaries and packages for various Linux distributions,
as well as [Go library package](https://pkg.go.dev/github.com/FerretDB/FerretDB/ferretdb) that embeds FerretDB into your application.
See [our documentation](https://docs.ferretdb.io/v1.24/quickstart-guide/) for more details.
We also provide binaries and packages for various Linux distributions.
as well as [Go library package](https://pkg.go.dev/github.com/FerretDB/FerretDB/v2/ferretdb)
that embeds FerretDB into your application.
See [our documentation](https://docs.ferretdb.io/installation/) for more details.
## Building and packaging
<!-- textlint-disable one-sentence-per-line -->
> [!NOTE]
> We strongly advise users not to build FerretDB themselves.
> We advise users not to build FerretDB themselves.
> Instead, use binaries, Docker images, or packages provided by us.
<!-- textlint-enable one-sentence-per-line -->
FerretDB could be built as any other Go program,
but a few generated files and build tags could affect it.
See [there](https://pkg.go.dev/github.com/FerretDB/FerretDB/build/version) for more details.
See [there](https://pkg.go.dev/github.com/FerretDB/FerretDB/v2/build/version) for more details.
## Managed FerretDB at cloud providers
- [FerretDB Cloud](https://cloud.ferretdb.com/)
- [Civo](https://www.civo.com/marketplace/FerretDB)
- [Tembo](https://tembo.io/docs/tembo-stacks/mongo-alternative)
- [Elestio](https://elest.io/open-source/ferretdb)
- [Cozystack](https://cozystack.io/docs/components/#managed-ferretdb).
## Documentation
- [Documentation for users](https://docs.ferretdb.io/).
- [Documentation for Go developers about embeddable FerretDB](https://pkg.go.dev/github.com/FerretDB/FerretDB/ferretdb).
- [Cozystack](https://cozystack.io/docs/components/#managed-ferretdb)
## Community
- Website and blog: https://www.ferretdb.com/.
- Twitter: [@ferret_db](https://twitter.com/ferret_db).
- Website: https://www.ferretdb.com/.
- Blog: https://blog.ferretdb.io/.
- Documentation: https://docs.ferretdb.io/.
- X (Twitter): [@ferret_db](https://x.com/ferret_db).
- Mastodon: [@ferretdb@techhub.social](https://techhub.social/@ferretdb).
- [Slack chat](https://join.slack.com/t/ferretdb/shared_invite/zt-zqe9hj8g-ZcMG3~5Cs5u9uuOPnZB8~A) for quick questions.
- [Slack chat](https://slack.ferretdb.io/) for quick questions.
- [GitHub Discussions](https://github.com/FerretDB/FerretDB/discussions) for longer topics.
- [GitHub Issues](https://github.com/FerretDB/FerretDB/issues) for bugs and missing features.
- [Open Office Hours meeting](https://calendar.google.com/calendar/event?action=TEMPLATE&tmeid=NGhrZTA5dXZ0MzQzN2gyaGVtZmx2aWxmN2pfMjAyNDA0MDhUMTcwMDAwWiBjX24zN3RxdW9yZWlsOWIwMm0wNzQwMDA3MjQ0QGc&tmsrc=c_n37tquoreil9b02m0740007244%40group.calendar.google.com&scp=ALL)
every Monday at 17:00 UTC at [Google Meet](https://meet.google.com/mcb-arhw-qbq).
If you want to contact FerretDB Inc., please use [this form](https://www.ferretdb.com/contact/).
+268 -257
View File
@@ -8,14 +8,17 @@ env:
vars:
SHARD_INDEX: 0
SHARD_TOTAL: 0
DISABLE_PUSHDOWN: false
TEST_RUN: ""
TEST_TIMEOUT: 35m
NO_XFAIL: false
BENCH_TIME: 5s
TESTJS_PORT: 27017
RACE_FLAG: -race={{and (ne OS "windows") (ne ARCH "arm") (ne ARCH "riscv64")}}
BUILD_TAGS: ferretdb_debug,ferretdb_hana
SERVICES: postgres postgres_secured mysql mongodb mongodb_secured jaeger
BUILD_TAGS: ferretdb_dev
MODE: diff-normal
SERVICES: jaeger mongodb mongodb-secure postgres yugabytedb
LLM: qwen3:0.6b # sync with code
LLM_CONTEXT: 40960 # https://ollama.com/library/qwen3:0.6b
tasks:
# invoked when `task` is run without arguments
@@ -30,7 +33,6 @@ tasks:
- task: test-unit
- task: lint
- task: security
- task: fmt-yaml
- task: docs
init-tools:
@@ -47,6 +49,13 @@ tasks:
- go mod verify
- go generate -x
init-tools-mcphost:
dir: tools/mcphost
cmds:
- go mod tidy
- go mod verify
- go generate -x
init-integration:
dir: integration
cmds:
@@ -55,7 +64,7 @@ tasks:
init:
desc: "Install development tools"
deps: [init-tools, init-tools-golangci, init-integration]
deps: [init-tools, init-tools-golangci, init-tools-mcphost, init-integration]
cmds:
- go mod tidy
- go mod verify
@@ -105,6 +114,15 @@ tasks:
- docker compose ps --all
- docker compose logs --follow
env-logs-collect:
desc: "Collect logs from Docker and save them to tmp/logs"
cmds:
- bin/envtool{{exeExt}} shell mkdir tmp/logs
- docker compose ps --all > tmp/logs/compose-ps.txt
- docker compose logs > tmp/logs/compose-logs.txt
- docker compose exec yugabytedb sh -c "bin/yugabyted collect_logs --stdout | gunzip | tar -xf - --one-top-level=/tmp/logs"
- docker compose cp yugabytedb:/tmp/logs tmp/logs/yugabytedb
env-up:
desc: "Start development environment"
deps: [env-up-detach, env-setup]
@@ -153,17 +171,18 @@ tasks:
desc: "Generate (and format) Go code"
cmds:
- go generate -x ./...
#- bin/deadcode{{exeExt}} -generated -test -filter=documentdb_api ./cmd/ferretdb
- bin/task{{exeExt}} -d integration integration-gen
- task: fmt-go
# invoked by FerretDB/github-actions/linters action
fmt:
deps: [fmt-go]
# invoked by FerretDB/github-actions/linters action
fmt-go:
desc: "Format Go code"
cmds:
- bin/goimports{{exeExt}} -local=github.com/FerretDB/FerretDB -w .
- bin/goimports{{exeExt}} -local=github.com/FerretDB/FerretDB/v2 -w .
- bin/gofumpt{{exeExt}} -w .
test:
@@ -173,27 +192,30 @@ tasks:
test-unit-short:
desc: "Run short unit tests (with caching)"
cmds:
- go test -short -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -coverpkg=./... -coverprofile=cover.txt ./...
- go test -c -o /dev/null ./...
- go test -short -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -coverprofile=cover.txt ./...
- bin/task{{exeExt}} -d tools tools-test-short
test-unit:
desc: "Run all unit tests"
cmds:
- go test -count=1 -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -coverpkg=./... -coverprofile=cover.txt ./...
- go test -c -o /dev/null ./...
- go test -v -count=1 -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -coverprofile=cover.txt ./... | tee unit.txt
- go test -count=1 -timeout={{.TEST_TIMEOUT}} {{.RACE_FLAG}} -tags={{.BUILD_TAGS}} -shuffle=on -bench=. -benchtime=1x ./...
- bin/task{{exeExt}} -d tools tools-test
test-integration-build:
run: once
dir: integration
cmd: go test -c -o /dev/null ./...
test-integration:
desc: "Run integration tests for several backends in parallel"
deps:
- test-integration-postgresql
- test-integration-sqlite
# no test-integration-hana
- test-integration-mongodb
desc: "Run integration tests in parallel"
deps: [test-integration-postgresql, test-integration-mongodb]
test-integration-postgresql:
desc: "Run integration tests for `postgresql` backend"
deps: [gen-version]
desc: "Run integration tests for PostgreSQL/DocumentDB"
deps: [gen-version, test-integration-build]
dir: integration
cmds:
- >
@@ -202,6 +224,7 @@ tasks:
--shard-total={{.SHARD_TOTAL}}
--run='{{.TEST_RUN}}'
--skip='{{.TEST_SKIP}}'
--raw-prefix=postgresql-{{.SHARD_INDEX}}-{{.SHARD_TOTAL}}
--
-count=1
-timeout={{.TEST_TIMEOUT}}
@@ -211,93 +234,14 @@ tasks:
-coverpkg=../...
-coverprofile=integration-postgresql.txt
./...
-target-backend=ferretdb-postgresql
-target-tls
-postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?search_path='
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=client.pem&tlsCaFile=rootCA-cert.pem&replicaSet=rs0'
-disable-pushdown={{.DISABLE_PUSHDOWN}}
test-integration-sqlite:
desc: "Run integration tests for `sqlite` backend"
deps: [gen-version]
dir: integration
cmds:
- >
../bin/envtool{{exeExt}} tests run
--shard-index={{.SHARD_INDEX}}
--shard-total={{.SHARD_TOTAL}}
--run='{{.TEST_RUN}}'
--skip='{{.TEST_SKIP}}'
--
-count=1
-timeout={{.TEST_TIMEOUT}}
{{.RACE_FLAG}}
-tags={{.BUILD_TAGS}}
-shuffle=on
-coverpkg=../...
-coverprofile=integration-sqlite.txt
./...
-target-backend=ferretdb-sqlite
-sqlite-url=file:../tmp/sqlite-tests/
-target-tls
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=client.pem&tlsCaFile=rootCA-cert.pem&replicaSet=rs0'
-disable-pushdown={{.DISABLE_PUSHDOWN}}
test-integration-mysql:
desc: "Run integration tests for `mysql` handler"
deps: [gen-version]
dir: integration
cmds:
- >
../bin/envtool{{exeExt}} tests run
--shard-index={{.SHARD_INDEX}}
--shard-total={{.SHARD_TOTAL}}
--run='{{.TEST_RUN}}'
--skip='{{.TEST_SKIP}}'
--
-count=1
-timeout={{.TEST_TIMEOUT}}
{{.RACE_FLAG}}
-tags={{.BUILD_TAGS}}
-shuffle=on
-coverpkg=../...
-coverprofile=integration-mysql.txt
./...
-target-backend=ferretdb-mysql
-target-tls
-mysql-url='mysql://username:password@127.0.0.1:3306/ferretdb'
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=client.pem&tlsCaFile=rootCA-cert.pem&replicaSet=rs0'
-disable-pushdown={{.DISABLE_PUSHDOWN}}
test-integration-hana:
desc: "Run integration tests for `hana` handler"
deps: [gen-version]
dir: integration
cmds:
- >
../bin/envtool{{exeExt}} tests run
--shard-index={{.SHARD_INDEX}}
--shard-total={{.SHARD_TOTAL}}
--run='{{.TEST_RUN}}'
--skip='{{.TEST_SKIP}}'
--
-count=1
-timeout={{.TEST_TIMEOUT}}
{{.RACE_FLAG}}
-tags={{.BUILD_TAGS}}
-shuffle=on
-coverpkg=../...
-coverprofile=integration-hana.txt
./...
-target-backend=ferretdb-hana
-target-tls
-hana-url=$FERRETDB_HANA_URL
-compat-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=client.pem&tlsCaFile=rootCA-cert.pem&replicaSet=rs0'
-disable-pushdown={{.DISABLE_PUSHDOWN}}
-target-backend=ferretdb
-postgresql-url='postgres://username:password@127.0.0.1:5432/postgres'
-compat-url='mongodb://username:password@127.0.0.1:47017/?replicaSet=rs0'
-no-xfail={{.NO_XFAIL}}
test-integration-mongodb:
desc: "Run integration tests for MongoDB"
deps: [gen-version]
deps: [gen-version, test-integration-build]
dir: integration
cmds:
- >
@@ -306,6 +250,7 @@ tasks:
--shard-total={{.SHARD_TOTAL}}
--run='{{.TEST_RUN}}'
--skip='{{.TEST_SKIP}}'
--raw-prefix=mongodb-{{.SHARD_INDEX}}-{{.SHARD_TOTAL}}
--
-count=1
-timeout={{.TEST_TIMEOUT}}
@@ -315,8 +260,50 @@ tasks:
-coverpkg=../...
-coverprofile=integration-mongodb.txt
./...
-target-url='mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=client.pem&tlsCaFile=rootCA-cert.pem&replicaSet=rs0'
-target-url='mongodb://username:password@127.0.0.1:47017/?replicaSet=rs0'
-target-backend=mongodb
-no-xfail={{.NO_XFAIL}}
test-integration-yugabytedb:
desc: "Run integration tests for FerretDB with YugabyteDB sequentially"
vars:
# DPanic on InternalError stops execution of remaining tests,
# ensure to complete all tests by not setting development build tag and without race detector.
# TODO https://github.com/FerretDB/FerretDB/issues/5397
BUILD_TAGS: ""
RACE_FLAG: -race=false
deps:
- task: gen-version
vars:
BUILD_TAGS:
ref: .BUILD_TAGS
RACE_FLAG:
ref: .RACE_FLAG
- task: test-integration-build
dir: integration
cmds:
- >
../bin/envtool{{exeExt}} tests run
--shard-index={{.SHARD_INDEX}}
--shard-total={{.SHARD_TOTAL}}
--run='{{.TEST_RUN}}'
--skip='{{.TEST_SKIP}}'
--raw-prefix=yugabytedb-{{.SHARD_INDEX}}-{{.SHARD_TOTAL}}
--
-count=1
-parallel=1
-p=1
-timeout={{.TEST_TIMEOUT}}
{{.RACE_FLAG}}
-tags={{.BUILD_TAGS}}
-shuffle=on
-coverpkg=../...
-coverprofile=integration-yugabytedb.txt
./...
-target-backend=ferretdb-yugabytedb
-postgresql-url='postgres://pg-user:pg-pass@127.0.0.1:5433/yugabyte'
-compat-url='mongodb://username:password@127.0.0.1:47017/?replicaSet=rs0'
-no-xfail={{.NO_XFAIL}}
bench-unit:
desc: "Run unit benchmarks"
@@ -326,124 +313,70 @@ tasks:
- bin/benchstat{{exeExt}} old.txt new.txt
run:
desc: "Run FerretDB with `postgresql` backend"
desc: "Run FerretDB without auth"
deps: [build-host]
cmds:
- >
bin/ferretdb{{exeExt}}
--mode={{.MODE}}
--no-auth
--listen-addr=:27017
--proxy-addr=127.0.0.1:47017
--mode=diff-normal
--repl-set-name=rs0
--handler=pg
--postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?search_path='
--test-records-dir=tmp/records
--listen-data-api-addr=127.0.0.1:8899
--listen-mcp-addr=127.0.0.1:8081
--proxy-addr=127.0.0.1:37017
--postgresql-url='postgres://username:password@127.0.0.1:5432/postgres'
--otel-traces-url=http://127.0.0.1:4318/v1/traces
--dev-records-dir=tmp/records
run-sqlite:
desc: "Run FerretDB with `sqlite` backend"
run-secure:
desc: "Run FerretDB with auth"
deps: [build-host]
cmds:
- bin/envtool{{exeExt}} shell mkdir tmp/sqlite
- >
bin/ferretdb{{exeExt}}
--mode={{.MODE}}
--listen-addr=:27017
--listen-data-api-addr=127.0.0.1:8899
--listen-mcp-addr=127.0.0.1:8081
--proxy-addr=127.0.0.1:47017
--mode=diff-normal
--handler=sqlite
--sqlite-url=file:tmp/sqlite/
--test-records-dir=tmp/records
--postgresql-url='postgres://username:password@127.0.0.1:5432/postgres'
--otel-traces-url=http://127.0.0.1:4318/v1/traces
--dev-repl-set-name=rs0
--dev-records-dir=tmp/records
run-mysql:
desc: "Run FerretDB with `mysql` backend"
run-yugabytedb:
desc: "Run FerretDB with YugabyteDB without auth"
deps: [build-host]
cmds:
- >
bin/ferretdb{{exeExt}}
--mode={{.MODE}}
--no-auth
--listen-addr=:27017
--proxy-addr=127.0.0.1:47017
--mode=diff-normal
--handler=mysql
--mysql-url='mysql://username@127.0.0.1:3306/ferretdb'
--test-records-dir=tmp/records
--listen-data-api-addr=127.0.0.1:8899
--listen-mcp-addr=127.0.0.1:8081
--proxy-addr=127.0.0.1:37017
--postgresql-url='postgres://username:password@127.0.0.1:5433/yugabyte'
--otel-traces-url=http://127.0.0.1:4318/v1/traces
--dev-records-dir=tmp/records
# set FERRETDB_HANA_URL environment variable to use it
run-hana:
desc: "Run FerretDB with `hana` backend"
run-yugabytedb-secure:
desc: "Run FerretDB with YugabyteDB with auth"
deps: [build-host]
cmds:
- >
bin/ferretdb{{exeExt}}
--mode={{.MODE}}
--listen-addr=:27017
--listen-data-api-addr=127.0.0.1:8899
--listen-mcp-addr=127.0.0.1:8081
--proxy-addr=127.0.0.1:47017
--mode=diff-normal
--handler=hana
--test-records-dir=tmp/records
run-secured:
desc: "Run FerretDB with `postgresql` backend (TLS, auth required)"
deps: [build-host]
cmds:
- >
bin/ferretdb{{exeExt}}
--listen-addr=''
--listen-tls=:27018
--listen-tls-cert-file=./build/certs/server-cert.pem
--listen-tls-key-file=./build/certs/server-key.pem
--listen-tls-ca-file=./build/certs/rootCA-cert.pem
--proxy-addr=127.0.0.1:47017
--mode=diff-normal
--handler=pg
--postgresql-url='postgres://127.0.0.1:5433/ferretdb?search_path='
--test-records-dir=tmp/records
run-proxy:
desc: "Run FerretDB in the diff-proxy mode"
deps: [build-host]
cmds:
- >
bin/ferretdb{{exeExt}}
--listen-addr=:27017
--proxy-addr=127.0.0.1:47017
--mode=diff-proxy
--handler=pg
--postgresql-url='postgres://username@127.0.0.1:5432/ferretdb?search_path='
--test-records-dir=tmp/records
run-sqlite-proxy:
desc: "Run FerretDB with `sqlite` handler in the diff-proxy mode"
deps: [build-host]
cmds:
- bin/envtool{{exeExt}} shell mkdir tmp/sqlite
- >
bin/ferretdb{{exeExt}}
--listen-addr=:27017
--proxy-addr=127.0.0.1:47017
--mode=diff-proxy
--handler=sqlite
--sqlite-url=file:tmp/sqlite/
--test-records-dir=tmp/records
run-proxy-secured:
desc: "Run FerretDB in the diff-proxy mode (TLS, auth required)"
deps: [build-host]
cmds:
- >
bin/ferretdb{{exeExt}}
--listen-addr=''
--listen-tls=:27018
--listen-tls-cert-file=./build/certs/server-cert.pem
--listen-tls-key-file=./build/certs/server-key.pem
--listen-tls-ca-file=./build/certs/rootCA-cert.pem
--proxy-addr=127.0.0.1:47018
--proxy-tls-cert-file=./build/certs/client-cert.pem
--proxy-tls-key-file=./build/certs/client-key.pem
--proxy-tls-ca-file=./build/certs/rootCA-cert.pem
--mode=diff-proxy
--handler=pg
--postgresql-url='postgres://username@127.0.0.1:5433/ferretdb?search_path='
--test-records-dir=tmp/records
--postgresql-url='postgres://username:password@127.0.0.1:5433/yugabyte'
--otel-traces-url=http://127.0.0.1:4318/v1/traces
--dev-repl-set-name=rs0
--dev-records-dir=tmp/records
# invoked by FerretDB/github-actions/linters action
lint:
desc: "Run linters"
cmds:
@@ -451,10 +384,9 @@ tasks:
- bin/envtool{{exeExt}} shell mkdir tmp/githubcache
- bin/golangci-lint{{exeExt}} run --config=.golangci.yml
- bin/golangci-lint{{exeExt}} run --config=.golangci-new.yml
- bin/go-consistent{{exeExt}} -pedantic ./cmd/... ./internal/... ./build/... ./ferretdb/...
- bin/go-consistent{{exeExt}} -pedantic ./cmd/... ./ferretdb/... ./internal/...
- go vet -vettool=bin/checkswitch{{exeExt}} ./...
# - go vet -vettool=bin/checkcomments{{exeExt}} ./...
- go vet -vettool=bin/checkcomments{{exeExt}} ./...
- bin/task{{exeExt}} -d integration lint
- bin/task{{exeExt}} -d tools lint
@@ -473,27 +405,41 @@ tasks:
security-trivy:
cmds:
- docker compose run --rm trivy --config=./build/trivy.yml filesystem .
- docker compose run --rm trivy --config=./build/trivy.yml filesystem --ignorefile=./build/.trivyignore .
godocs:
desc: "Serve Go code documentation"
cmds:
- bin/pkgsite{{exeExt}} -http=127.0.0.1:6060 -open
ollama:
desc: "Run local Ollama server"
cmds:
- ollama serve
env:
OLLAMA_CONTEXT_LENGTH: "{{.LLM_CONTEXT}}"
OLLAMA_DEBUG: 2
ollama-model:
desc: "Pull and load Ollama model"
cmds:
- ollama pull {{.LLM}}
- ollama run --think=false {{.LLM}} 'Hi!'
mcphost:
desc: "Run MCPHost for local Ollama server"
cmds:
- bin/mcphost --config=build/mcphost.json --model=ollama:{{.LLM}}
psql:
desc: "Run psql"
cmds:
- docker compose exec -e PGPASSWORD=password postgres psql -U username -d ferretdb
- docker compose exec postgres psql -p 5432 -U username -d postgres
sqlite3:
desc: "Run sqlite3"
ysqlsh:
desc: "Run YugabyteDB SQL shell"
cmds:
- sqlite3 tmp/sqlite/*.sqlite
mysql:
desc: "Run mysql"
cmds:
- docker compose exec -e MYSQL_PASSWORD=password mysql mysql -u username -ppassword ferretdb
- docker compose exec yugabytedb ./bin/ysqlsh postgres://username:password@host.docker.internal:5433/yugabyte
mongosh:
desc: "Run MongoDB shell (`mongosh`)"
@@ -503,17 +449,32 @@ tasks:
--verbose --eval 'disableTelemetry()' --shell
'mongodb://host.docker.internal:27017/?heartbeatFrequencyMS=300000'
mongosh-secured:
desc: "Run MongoDB shell (`mongosh`) with TLS and auth"
mongosh-secure:
desc: "Run MongoDB shell (`mongosh`) with auth"
cmds:
- >
docker compose exec mongodb mongosh
--verbose --eval 'disableTelemetry()' --shell
'mongodb://username:password@host.docker.internal:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem'
'mongodb://username:password@host.docker.internal:27017/?heartbeatFrequencyMS=300000'
mongo:
desc: "Run legacy MongoDB shell (`mongo`)"
cmds:
- >
docker compose run --rm legacy-mongo-shell
'mongodb://username:password@host.docker.internal:27017/'
testsql:
desc: "Run pgTAP with test.sql script"
cmds:
- docker compose exec postgres pg_prove --ext .sql --verbose -p 5432 -U username -d postgres /testscripts/
testjs:
desc: "Run legacy MongoDB shell (`mongo`) with test.js script"
cmds:
# Authentication with username:password will not work
# because this user is created by setup with a different salt length,
# and there are no other credentials to use.
- >
docker compose run --rm legacy-mongo-shell
'mongodb://host.docker.internal:{{.TESTJS_PORT}}/'
@@ -529,7 +490,6 @@ tasks:
--bootstrap=true
--use=false
--config=./build/buildkitd.toml
--driver=docker-container
--driver-opt network=host
--driver-opt env.JAEGER_TRACE=127.0.0.1:6831
--driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=-1
@@ -578,7 +538,7 @@ tasks:
vars: [DOCKER_ARCH, DIR_ARCH, BIN_FILE, BIN_DIR]
build-development:
desc: "Build development debug binaries"
desc: "Build development binaries"
cmds:
- echo 'bin-development' > build/version/package.txt
- bin/envtool{{exeExt}} shell rmdir tmp/build
@@ -587,18 +547,19 @@ tasks:
vars:
FILE: ferretdb/development
TARGET: development-binary
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=local,dest=tmp/build
- for: [amd64, arm64, arm/v7]
- for: [amd64, arm64]
task: build-move-check
vars:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
DOCKER_ARCH: "{{.ITEM}}" # arm/v7
DIR_ARCH: '{{replace "/" "_" .ITEM}}' # arm/v7 -> arm_v7
BIN_FILE: 'ferretdb-dev-linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
BIN_FILE: 'ferretdb-dev-{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
BIN_DIR: bin-dev
build-production:
desc: "Build production non-debug binaries"
desc: "Build production binaries"
cmds:
- echo 'bin' > build/version/package.txt
- bin/envtool{{exeExt}} shell rmdir tmp/build
@@ -607,14 +568,15 @@ tasks:
vars:
FILE: ferretdb/production
TARGET: production-binary
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=local,dest=tmp/build
- for: [amd64, arm64, arm/v7]
- for: [amd64, arm64]
task: build-move-check
vars:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
DOCKER_ARCH: "{{.ITEM}}" # arm/v7
DIR_ARCH: '{{replace "/" "_" .ITEM}}' # arm/v7 -> arm_v7
BIN_FILE: 'ferretdb-linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
BIN_FILE: 'ferretdb-{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
BIN_DIR: bin
docker-local:
@@ -631,14 +593,27 @@ tasks:
vars: [FILE]
# TODO https://github.com/FerretDB/FerretDB/issues/2212
docker-all-in-one-push:
docker-eval-dev-push:
cmds:
- echo 'docker-all-in-one' > build/version/package.txt
- echo 'docker-eval-dev' > build/version/package.txt
- task: docker-build
vars:
FILE: ferretdb/all-in-one
TARGET: all-in-one
PLATFORM: linux/amd64,linux/arm64 # no mongosh for arm/v6 and arm/v7
FILE: ferretdb/eval-dev
TARGET: eval-dev
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=image,push=true
requires:
vars: [DOCKER_IMAGES]
# TODO https://github.com/FerretDB/FerretDB/issues/2212
docker-eval-push:
cmds:
- echo 'docker-eval' > build/version/package.txt
- task: docker-build
vars:
FILE: ferretdb/eval
TARGET: eval
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=image,push=true
requires:
vars: [DOCKER_IMAGES]
@@ -651,7 +626,7 @@ tasks:
vars:
FILE: ferretdb/development
TARGET: development
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=image,push=true
requires:
vars: [DOCKER_IMAGES]
@@ -664,7 +639,7 @@ tasks:
vars:
FILE: ferretdb/production
TARGET: production
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=image,push=true
requires:
vars: [DOCKER_IMAGES]
@@ -686,6 +661,7 @@ tasks:
--packager={{.PACKAGER}}
--target=tmp/{{.PACKAGER}}s/ferretdb-{{.FILE_SUFFIX}}.{{.PACKAGER}}
env:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
PACKAGE_VERSION: "{{.PACKAGE_VERSION}}"
PACKAGE_ARCH: "{{.PACKAGE_ARCH}}"
DIR_ARCH: "{{.DIR_ARCH}}"
@@ -713,21 +689,22 @@ tasks:
vars:
FILE: ferretdb/development
TARGET: development-binary
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=local,dest=tmp/build
- for: [amd64, arm64, arm/v7]
- for: [amd64, arm64]
task: packages-build
vars:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
PACKAGER: deb
PACKAGE_VERSION: "{{.PACKAGE_VERSION}}"
PACKAGE_ARCH: '{{replace "/" "" .ITEM | replace "v" ""}}' # arm/v7 -> arm7
DIR_ARCH: '{{replace "/" "_" .ITEM}}' # arm/v7 -> arm_v7
FILE_SUFFIX: 'dev-linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
- for: [amd64, arm64, arm/v7]
FILE_SUFFIX: 'dev-{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
- for: [amd64, arm64]
task: packages-deb-check
vars:
DOCKER_ARCH: "{{.ITEM}}" # arm/v7
FILE_SUFFIX: 'dev-linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
FILE_SUFFIX: 'dev-{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
vars:
PACKAGE_VERSION:
sh: bin/envtool{{exeExt}} package-version
@@ -741,21 +718,22 @@ tasks:
vars:
FILE: ferretdb/production
TARGET: production-binary
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=local,dest=tmp/build
- for: [amd64, arm64, arm/v7]
- for: [amd64, arm64]
task: packages-build
vars:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
PACKAGER: deb
PACKAGE_VERSION: "{{.PACKAGE_VERSION}}"
PACKAGE_ARCH: '{{replace "/" "" .ITEM | replace "v" ""}}' # arm/v7 -> arm7
DIR_ARCH: '{{replace "/" "_" .ITEM}}' # arm/v7 -> arm_v7
FILE_SUFFIX: 'linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
- for: [amd64, arm64, arm/v7]
FILE_SUFFIX: '{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
- for: [amd64, arm64]
task: packages-deb-check
vars:
DOCKER_ARCH: "{{.ITEM}}" # arm/v7
FILE_SUFFIX: 'linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
FILE_SUFFIX: '{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
vars:
PACKAGE_VERSION:
sh: bin/envtool{{exeExt}} package-version
@@ -781,21 +759,22 @@ tasks:
vars:
FILE: ferretdb/development
TARGET: development-binary
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=local,dest=tmp/build
- for: [amd64, arm64, arm/v7]
- for: [amd64, arm64]
task: packages-build
vars:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
PACKAGER: rpm
PACKAGE_VERSION: "{{.PACKAGE_VERSION}}"
PACKAGE_ARCH: '{{replace "/" "" .ITEM | replace "v" ""}}' # arm/v7 -> arm7
DIR_ARCH: '{{replace "/" "_" .ITEM}}' # arm/v7 -> arm_v7
FILE_SUFFIX: 'dev-linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
- for: [amd64, arm64, arm/v7]
FILE_SUFFIX: 'dev-{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
- for: [amd64, arm64]
task: packages-rpm-check
vars:
DOCKER_ARCH: "{{.ITEM}}" # arm/v7
FILE_SUFFIX: 'dev-linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
FILE_SUFFIX: 'dev-{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
vars:
PACKAGE_VERSION:
sh: bin/envtool{{exeExt}} package-version
@@ -809,21 +788,22 @@ tasks:
vars:
FILE: ferretdb/production
TARGET: production-binary
PLATFORM: linux/amd64,linux/arm64,linux/arm/v7 # TODO https://github.com/FerretDB/FerretDB/issues/3354
PLATFORM: linux/amd64,linux/arm64
OUTPUT: type=local,dest=tmp/build
- for: [amd64, arm64, arm/v7]
- for: [amd64, arm64]
task: packages-build
vars:
# TODO https://github.com/FerretDB/FerretDB/issues/5019
PACKAGER: rpm
PACKAGE_VERSION: "{{.PACKAGE_VERSION}}"
PACKAGE_ARCH: '{{replace "/" "" .ITEM | replace "v" ""}}' # arm/v7 -> arm7
DIR_ARCH: '{{replace "/" "_" .ITEM}}' # arm/v7 -> arm_v7
FILE_SUFFIX: 'linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
- for: [amd64, arm64, arm/v7]
FILE_SUFFIX: '{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
- for: [amd64, arm64]
task: packages-rpm-check
vars:
DOCKER_ARCH: "{{.ITEM}}" # arm/v7
FILE_SUFFIX: 'linux-{{replace "/" "" .ITEM}}' # arm/v7 -> armv7
FILE_SUFFIX: '{{replace "/" "" .ITEM}}-linux' # arm/v7 -> armv7
vars:
PACKAGE_VERSION:
sh: bin/envtool{{exeExt}} package-version
@@ -833,6 +813,14 @@ tasks:
cmds:
- docker compose run --rm prettier --write --print-width 120 "**/*.js"
# .js files are also used for mongosh output snippets
# (with added `response = ` to make them valid JavaScript).
# The formatting of quotes, commas and semicolons should not be modified.
# Flags are set to ensure that.
- >
docker compose run --rm prettier --write --single-quote --trailing-comma none --no-semi
--print-width 120 "website/*docs/**/*.js"
fmt-json:
desc: "Format JSON files"
cmds:
@@ -843,29 +831,52 @@ tasks:
cmds:
- docker compose run --rm prettier --write --print-width 120 "**/*.yml"
# see https://github.com/DavidAnson/markdownlint-cli2#command-line for the reason we use double-quotes
# See https://github.com/DavidAnson/markdownlint-cli2#command-line for the reason we use double quotes in command.
fmt-docs:
desc: "Format and lint documentation"
deps: [fmt-js, fmt-json, fmt-yaml]
deps: [gen-version, fmt-js, fmt-json, fmt-yaml]
run: once
cmds:
- >
docker compose run --rm textlint --fix --config build/.textlintrc --ignore-path .prettierignore
"**/*.md" "**/*.mdx" ".github/**/*.md"
# Use the same flags as in fmt-js for mongosh snippets above.
#
# Note that embedded code snippets in ``` Markdown blocks are not reformatted if they can't be parsed.
# Ensure that the code is valid and can be parsed by changing formatting manually
# and running this task again, checking that formatting changes.
#
# TODO --print-width 120 (https://github.com/FerretDB/FerretDB/issues/4636)
- >
docker compose run --rm prettier --write --parser markdown --no-semi --single-quote --trailing-comma none
docker compose run --rm prettier --write --single-quote --trailing-comma none --no-semi
"**/*.md" "**/*.mdx"
- >
docker compose run --rm markdownlint --config "build/.markdownlint.yml"
"**/*.md" "**/*.mdx"
# - bin/checkdocs
"**/*.md" "**/*.mdx" "#build/postgres-documentdb/documentdb/"
- bin/envtool{{exeExt}} shell rmdir tmp/githubcache
- bin/envtool{{exeExt}} shell mkdir tmp/githubcache
- bin/checkdocs
docs-fmt:
deps: [fmt-docs]
docs-gen:
desc: "Generate documentation examples using CTS tool"
cmds:
- bin/opendocdb-cts fmt --dir=website/docs/guides
- bin/opendocdb-cts convert --dir=website/docs/guides website/docs/guides --db=db
- task: fmt-docs
cts:
desc: "Run CTS tests against FerretDB"
cmds:
- bin/opendocdb-cts run --dir=website/docs/guides --uri=mongodb://127.0.0.1:27017/cts
docs:
desc: "Format, lint and build documentation"
deps: [fmt-docs]
deps: [docs-gen]
cmds:
- docker compose run --rm docusaurus-docs build
@@ -877,17 +888,17 @@ tasks:
pngcrush:
cmds:
- pngcrush -ow -brute {{.FILE}}
- pngcrush -ow -brute -rem text {{.FILE}}
requires:
vars: [FILE]
pngcrush-new:
cmds:
- git diff --name-only upstream/main '*.png' | parallel "pngcrush -ow -brute {} {.}.temp"
- git diff --name-only upstream/main '*.png' | parallel "pngcrush -ow -brute -rem text {} {.}.temp"
pngcrush-all:
cmds:
- find . -type f -name '*.png' | parallel "pngcrush -ow -brute {} {.}.temp"
- find . -type f -name '*.png' | parallel "pngcrush -ow -brute -rem text {} {.}.temp"
docs-dev:
desc: "Start documentation development server"
@@ -925,8 +936,8 @@ tasks:
- docker compose run --rm --service-ports wrangler pages deployment create --project-name=blog website/build
changelog:
desc: "Generate changelogs locally for a given milestone."
desc: "Generate changelog"
cmds:
- bin/generatechangelog -milestone={{.MILESTONE_TITLE}} -previous={{.PREVIOUS_MILESTONE_TITLE}}
- bin/generatechangelog -prev={{.PREV}} -next={{.NEXT}}
requires:
vars: [MILESTONE_TITLE, PREVIOUS_MILESTONE_TITLE]
vars: [PREV, NEXT]
+3
View File
@@ -11,4 +11,7 @@ MD006: false # deprecated
MD013: false # we use https://sembr.org that could lead to long lines
MD026: { punctuation: ".,;:。,;:" } # default is .,;:!。,;:!
MD029: { style: ordered }
MD033: false # for MDX
MD034: false # autolink tags <link> no longer supported by MDX
MD037: false # clashes with MDX comments
MD059: false # not yet
+7
View File
@@ -0,0 +1,7 @@
# https://github.com/cloudwego/eino#dependencies
# TODO https://github.com/cloudwego/eino/issues/291
CVE-2025-30153
# https://nvd.nist.gov/vuln/detail/CVE-2025-51471
# https://github.com/ollama/ollama/pull/10750
CVE-2025-51471
-1
View File
@@ -1 +0,0 @@
FROM citusdata/citus:13.0
+1 -1
View File
@@ -1 +1 @@
FROM ghcr.io/ferretdb/ferretdb-prettier:3.5.3-1
FROM ghcr.io/ferretdb/ferretdb-prettier:3.6.2-1
+1 -1
View File
@@ -1 +1 @@
FROM ghcr.io/ferretdb/ferretdb-textlint:14.7.1-1
FROM ghcr.io/ferretdb/ferretdb-textlint:15.2.0-1
+1 -1
View File
@@ -1 +1 @@
FROM ghcr.io/ferretdb/ferretdb-wrangler:4.7.1-1
FROM ghcr.io/ferretdb/ferretdb-wrangler:4.24.0-1
+1 -1
View File
@@ -1 +1 @@
FROM jaegertracing/all-in-one:1.69.0
FROM jaegertracing/all-in-one:1.74.0
+1
View File
@@ -1 +1,2 @@
# TODO https://github.com/FerretDB/FerretDB/issues/5073
FROM ghcr.io/ferretdb/legacy-mongo-shell:7.0.8-1
+1 -1
View File
@@ -1 +1 @@
FROM davidanson/markdownlint-cli2:v0.13.0
FROM davidanson/markdownlint-cli2:v0.18.1
-1
View File
@@ -1 +0,0 @@
FROM mongo:7.0.8
+16
View File
@@ -0,0 +1,16 @@
# syntax=docker/dockerfile:1
# TODO https://github.com/FerretDB/FerretDB/issues/5073
FROM mongo:7.0.8
# If you encounter an "unknown instruction" error there,
# please update Docker to the latest version.
# see https://github.com/docker-library/mongo/issues/475
RUN <<EOF
set -ex
echo 'topsecret' > /etc/mongod_keyfile.txt
chmod 0400 /etc/mongod_keyfile.txt
chown 999:999 /etc/mongod_keyfile.txt
EOF
-1
View File
@@ -1 +0,0 @@
FROM mysql:8.4.0
@@ -0,0 +1,5 @@
# Use development image and full tag close to the release.
FROM ghcr.io/ferretdb/postgres-documentdb-dev:17-0.107.0-ferretdb-2.7.0
# Use moving development image during development.
# FROM ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb
-1
View File
@@ -1 +0,0 @@
FROM postgres:16.9
+1 -1
View File
@@ -1,2 +1,2 @@
FROM aquasec/trivy:0.62.1
FROM aquasec/trivy:0.67.2
WORKDIR /workdir
+1
View File
@@ -0,0 +1 @@
FROM yugabytedb/yugabyte:2025.1.1.0-b197
+27
View File
@@ -0,0 +1,27 @@
# Use production image and full tag close to the release.
FROM --platform=linux/amd64 ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0
# Use moving development image during development.
# FROM --platform=linux/amd64 ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb
RUN --mount=type=cache,target=/var/cache/apt <<EOF
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
curl -L https://pgp.mongodb.com/server-7.0.asc | apt-key add -
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/debian bookworm/mongodb-org/7.0 main" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list
apt update
apt install -y git-lfs mongodb-mongosh tmux nano
cd /tmp
curl -OJL https://go.dev/dl/go1.25.4.linux-amd64.tar.gz
tar -xf go1.25.4.linux-amd64.tar.gz
rm go1.25.4.linux-amd64.tar.gz
mv go /usr/local/
ln -s /usr/local/go/bin/* /usr/local/bin/
go version
EOF
WORKDIR /ferretdb/build/ferretdb-bw
+26
View File
@@ -0,0 +1,26 @@
all: ferretdb_loader ferretdb
ferretdb_loader:
$(MAKE) -C $@
ferretdb:
$(MAKE) -C $@
clean:
$(MAKE) -C ferretdb_loader clean
$(MAKE) -C ferretdb clean
install:
$(MAKE) -C ferretdb_loader install
$(MAKE) -C ferretdb install
docker:
docker buildx build --builder=ferretdb --tag=ferretdb-bw --load .
docker run --platform=linux/amd64 --name=ferretdb-bw --rm -it \
-e POSTGRES_USER=username -e POSTGRES_PASSWORD=password -e POSTGRES_DB=postgres \
-v $(shell pwd)/../..:/ferretdb ferretdb-bw /bin/bash
docker-exec:
docker exec -it ferretdb-bw /bin/bash
.PHONY: all ferretdb_loader ferretdb
+4
View File
@@ -0,0 +1,4 @@
# FerretDB as a PostgreSQL background worker
Not supported.
Do not use yet.
+21
View File
@@ -0,0 +1,21 @@
GO ?= $(shell which go)
PG_CONFIG ?= pg_config
PG_INCLUDE := $(shell $(PG_CONFIG) --includedir-server)
PG_PKGLIB := $(shell $(PG_CONFIG) --pkglibdir)
INSTALL_BIN = $(shell which install)
CGO_CFLAGS ?= "-I$(PG_INCLUDE) -Wno-implicit-function-declaration"
CGO_LDFLAGS ?= "-shared"
object = ferretdb.so
all: $(object)
$(object): ferretdb.go
CGO_CFLAGS=$(CGO_CFLAGS) CGO_LDFLAGS=$(CGO_LDFLAGS) $(GO) build -v -tags=ferretdb_bw -buildmode=c-shared -o $@ .
install: $(object)
$(INSTALL_BIN) -c -m 755 $(object) $(PG_PKGLIB)
clean:
rm -f $(object)
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2021 FerretDB Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ferretdb_bw
package main
/*
#include "postgres.h"
*/
import "C"
import (
"context"
"log"
"log/slog"
"os"
"github.com/FerretDB/FerretDB/v2/build/version"
"github.com/FerretDB/FerretDB/v2/ferretdb"
"github.com/FerretDB/FerretDB/v2/internal/util/ctxutil"
)
//export BackgroundWorkerMain
func BackgroundWorkerMain(args C.Datum) {
log.SetPrefix("ferretdb: ")
log.SetFlags(0)
ctx, stop := ctxutil.SigTerm(context.Background())
defer stop()
f, err := ferretdb.New(&ferretdb.Config{
// TODO https://github.com/FerretDB/FerretDB/issues/4771
PostgreSQLURL: "postgres://username:password@127.0.0.1:5432/postgres",
ListenAddr: "127.0.0.1:27017",
StateDir: ".",
LogLevel: slog.LevelDebug,
LogOutput: os.Stderr,
})
if err != nil {
log.Fatal(err)
}
version.Get().Package = "extension"
done := make(chan struct{})
// Use WaitLatch, ResetLatch, and WL_POSTMASTER_DEATH to check if `postmaster` process died and call stop().
// TODO https://github.com/FerretDB/FerretDB/issues/4771
go func() {
f.Run(ctx)
close(done)
}()
uri := f.MongoDBURI()
log.Printf("Running at %s", uri)
<-done
return
}
func main() {
panic("not reached")
}
+86
View File
@@ -0,0 +1,86 @@
/* Code generated by cmd/cgo; DO NOT EDIT. */
/* package github.com/FerretDB/FerretDB/v2/build/ferretdb-bw/ferretdb */
#line 1 "cgo-builtin-export-prolog"
#include <stddef.h>
#ifndef GO_CGO_EXPORT_PROLOGUE_H
#define GO_CGO_EXPORT_PROLOGUE_H
#ifndef GO_CGO_GOSTRING_TYPEDEF
typedef struct { const char *p; ptrdiff_t n; } _GoString_;
#endif
#endif
/* Start of preamble from import "C" comments. */
#line 3 "ferretdb.go"
#include "postgres.h"
#line 1 "cgo-generated-wrapper"
/* End of preamble from import "C" comments. */
/* Start of boilerplate cgo prologue. */
#line 1 "cgo-gcc-export-header-prolog"
#ifndef GO_CGO_PROLOGUE_H
#define GO_CGO_PROLOGUE_H
typedef signed char GoInt8;
typedef unsigned char GoUint8;
typedef short GoInt16;
typedef unsigned short GoUint16;
typedef int GoInt32;
typedef unsigned int GoUint32;
typedef long long GoInt64;
typedef unsigned long long GoUint64;
typedef GoInt64 GoInt;
typedef GoUint64 GoUint;
typedef size_t GoUintptr;
typedef float GoFloat32;
typedef double GoFloat64;
#ifdef _MSC_VER
#include <complex.h>
typedef _Fcomplex GoComplex64;
typedef _Dcomplex GoComplex128;
#else
typedef float _Complex GoComplex64;
typedef double _Complex GoComplex128;
#endif
/*
static assertion to make sure the file is being used on architecture
at least with matching size of GoInt.
*/
typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1];
#ifndef GO_CGO_GOSTRING_TYPEDEF
typedef _GoString_ GoString;
#endif
typedef void *GoMap;
typedef void *GoChan;
typedef struct { void *t; void *v; } GoInterface;
typedef struct { void *data; GoInt len; GoInt cap; } GoSlice;
#endif
/* End of boilerplate cgo prologue. */
#ifdef __cplusplus
extern "C" {
#endif
extern void BackgroundWorkerMain(Datum args);
#ifdef __cplusplus
}
#endif
@@ -0,0 +1,4 @@
MODULES = ferretdb_loader
PG_CONFIG ?= pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
@@ -0,0 +1,53 @@
#include <dlfcn.h>
#include "postgres.h"
#include "pgstat.h"
#include "postmaster/bgworker.h"
#include "storage/ipc.h"
#include "storage/latch.h"
#include "storage/proc.h"
#include "miscadmin.h"
#include "utils/elog.h"
#include "fmgr.h"
PG_MODULE_MAGIC;
static char *ferretdb_so = "ferretdb.so";
PGDLLEXPORT void background_main(Datum main_arg);
void background_main(Datum main_arg)
{
bgworker_main_type f;
char *path;
void *h;
BackgroundWorkerUnblockSignals();
path = (char *)palloc(strlen(pkglib_path) + 1 + strlen(ferretdb_so) + 1);
join_path_components(path, pkglib_path, ferretdb_so);
elog(LOG, "ferretdb_loader: loading '%s'", path);
h = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
pfree(path);
f = (bgworker_main_type)dlsym(h, "BackgroundWorkerMain");
f(main_arg);
dlclose(h);
proc_exit(0);
}
void _PG_init(void)
{
BackgroundWorker worker;
MemSet(&worker, 0, sizeof(BackgroundWorker));
snprintf(worker.bgw_name, BGW_MAXLEN, "FerretDBLoader");
worker.bgw_flags = BGWORKER_SHMEM_ACCESS;
worker.bgw_start_time = BgWorkerStart_RecoveryFinished;
worker.bgw_restart_time = BGW_NEVER_RESTART;
snprintf(worker.bgw_library_name, BGW_MAXLEN, "ferretdb_loader");
snprintf(worker.bgw_function_name, BGW_MAXLEN, "background_main");
RegisterBackgroundWorker(&worker);
}
+2 -2
View File
@@ -9,9 +9,9 @@ ExecStart=/usr/bin/ferretdb
Restart=on-failure
# Configure the FerretDB service with `systemctl edit ferretdb`.
# For more configuration options check https://docs.ferretdb.io/v1.24/configuration/flags/
# For more configuration options check https://docs.ferretdb.io/configuration/flags/
Environment="FERRETDB_POSTGRESQL_URL=postgres://127.0.0.1:5432/ferretdb"
Environment="FERRETDB_POSTGRESQL_URL=postgres://127.0.0.1:5432/postgres"
[Install]
WantedBy=multi-user.target
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
set -ex
exec /usr/bin/runsvdir /etc/service
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
set -ex
exec /ferretdb
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
set -ex
exec /usr/local/bin/docker-entrypoint.sh postgres
+15 -18
View File
@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
# for development releases (`ferret-dev` image)
# for development releases (`ferretdb-dev` image)
# While we already know commit and version from commit.txt and version.txt inside image,
# it is not possible to use them in LABELs for the final image.
@@ -12,9 +12,10 @@ ARG LABEL_COMMIT
# prepare stage
FROM --platform=$BUILDPLATFORM golang:1.24.3 AS development-prepare
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM --platform=$BUILDPLATFORM golang:1.25.4-bookworm AS development-prepare
# use a single directory for all Go caches to simpliy RUN --mount commands below
# use a single directory for all Go caches to simplify RUN --mount commands below
ENV GOPATH=/cache/gopath
ENV GOCACHE=/cache/gocache
ENV GOMODCACHE=/cache/gomodcache
@@ -36,10 +37,10 @@ EOF
# build stage
FROM golang:1.24.3 AS development-build
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM golang:1.25.4-bookworm AS development-build
ARG TARGETARCH
ARG TARGETVARIANT
ARG LABEL_VERSION
ARG LABEL_COMMIT
@@ -66,18 +67,15 @@ set -ex
git status
# Do not raise it without providing a separate v1 build
# because v2+ is problematic for some virtualization platforms and older hardware.
# Do not raise without providing separate builds with those values
# because higher versions are problematic for some virtualization platforms and older hardware.
export GOAMD64=v1
# Set GOARM explicitly due to https://github.com/docker-library/golang/issues/494.
export GOARM=${TARGETVARIANT#v}
export GOARM64=v8.0
export CGO_ENABLED=1
# Disable race detector on arm64 due to https://github.com/golang/go/issues/29948
# (and that happens on GitHub-hosted Actions runners).
# Also disable it on arm/v6 and arm/v7 because it is not supported there.
RACE=false
if test "$TARGETARCH" = "amd64"
then
@@ -91,7 +89,7 @@ go env
# check if stdlib was cached
go install -v -race=$RACE std
go build -v -o=bin/ferretdb -race=$RACE -tags=ferretdb_debug -coverpkg=./... ./cmd/ferretdb
go build -v -o=bin/ferretdb -race=$RACE -tags=ferretdb_dev -coverpkg=./... ./cmd/ferretdb
go version -m bin/ferretdb
bin/ferretdb --version
@@ -107,7 +105,8 @@ COPY --from=development-build /src/bin/ferretdb /ferretdb
# final stage
FROM golang:1.24.3 AS development
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM golang:1.25.4-bookworm AS development
ENV GOCOVERDIR=/tmp/cover
ENV GORACE=halt_on_error=1,history_size=2
@@ -116,7 +115,7 @@ RUN mkdir /tmp/cover
COPY --from=development-build /src/bin/ferretdb /ferretdb
ENTRYPOINT ["/ferretdb"]
ENTRYPOINT [ "/ferretdb" ]
HEALTHCHECK --interval=1m --timeout=5s --retries=1 --start-period=30s --start-interval=5s \
CMD ["/ferretdb", "ping"]
@@ -130,17 +129,15 @@ ENV FERRETDB_LISTEN_ADDR=:27017
# ENV FERRETDB_LISTEN_TLS=:27018
ENV FERRETDB_DEBUG_ADDR=:8088
ENV FERRETDB_STATE_DIR=/state
ENV FERRETDB_SQLITE_URL=file:/state/
ARG LABEL_VERSION
ARG LABEL_COMMIT
# TODO https://github.com/FerretDB/FerretDB/issues/2212
LABEL org.opencontainers.image.description="A truly Open Source MongoDB alternative"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.description="A truly Open Source MongoDB alternative (development image)"
LABEL org.opencontainers.image.revision="${LABEL_COMMIT}"
LABEL org.opencontainers.image.source="https://github.com/FerretDB/FerretDB"
LABEL org.opencontainers.image.title="FerretDB"
LABEL org.opencontainers.image.title="FerretDB (development image)"
LABEL org.opencontainers.image.url="https://www.ferretdb.com/"
LABEL org.opencontainers.image.vendor="FerretDB Inc."
LABEL org.opencontainers.image.version="${LABEL_VERSION}"
@@ -1,8 +1,6 @@
# syntax=docker/dockerfile:1
# Dockerfile for all-in-one releases (`all-in-one` image).
# It always uses native compilation with race detector
# because packages are only available for amd64 and arm64 anyway.
# evaluation image with development release (`ferretdb-eval-dev` image)
# While we already know commit and version from commit.txt and version.txt inside image,
# it is not possible to use them in LABELs for the final image.
@@ -14,9 +12,10 @@ ARG LABEL_COMMIT
# prepare stage
FROM --platform=$BUILDPLATFORM golang:1.24.3 AS all-in-one-prepare
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM --platform=$BUILDPLATFORM golang:1.25.4-bookworm AS eval-dev-prepare
# use a single directory for all Go caches to simpliy RUN --mount commands below
# use a single directory for all Go caches to simplify RUN --mount commands below
ENV GOPATH=/cache/gopath
ENV GOCACHE=/cache/gocache
ENV GOMODCACHE=/cache/gomodcache
@@ -38,7 +37,8 @@ EOF
# build stage
FROM golang:1.24.3 AS all-in-one-build
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM golang:1.25.4-bookworm AS eval-dev-build
ARG TARGETARCH
@@ -60,24 +60,22 @@ WORKDIR /src
COPY . .
# to add a dependency
COPY --from=all-in-one-prepare /src/go.mod /src/go.sum /src/
COPY --from=eval-dev-prepare /src/go.mod /src/go.sum /src/
RUN --mount=type=cache,target=/cache <<EOF
set -ex
git status
# Do not raise it without providing a separate v1 build
# because v2+ is problematic for some virtualization platforms and older hardware.
# Do not raise without providing separate builds with those values
# because higher versions are problematic for some virtualization platforms and older hardware.
export GOAMD64=v1
# GOARM is not set because it is ignored for arm64
export GOARM64=v8.0
export CGO_ENABLED=1
# Disable race detector on arm64 due to https://github.com/golang/go/issues/29948
# (and that happens on GitHub-hosted Actions runners).
# Also disable it on arm/v6 and arm/v7 because it is not supported there.
RACE=false
if test "$TARGETARCH" = "amd64"
then
@@ -91,7 +89,7 @@ go env
# check if stdlib was cached
go install -v -race=$RACE std
go build -v -o=bin/ferretdb -race=$RACE -tags=ferretdb_debug -coverpkg=./... ./cmd/ferretdb
go build -v -o=bin/ferretdb -race=$RACE -tags=ferretdb_dev -coverpkg=./... ./cmd/ferretdb
go version -m bin/ferretdb
bin/ferretdb --version
@@ -100,66 +98,52 @@ EOF
# final stage
FROM postgres:16.9 AS all-in-one
# Use development image and full tag close to the release.
FROM ghcr.io/ferretdb/postgres-documentdb-dev:17-0.107.0-ferretdb-2.7.0 AS eval-dev
COPY --from=all-in-one-build /src/bin/ferretdb /ferretdb
ENV GOCOVERDIR=/tmp/cover
ENV GORACE=halt_on_error=1,history_size=2
RUN mkdir /tmp/cover
# all-in-one hacks start there
COPY --from=all-in-one-build /src/build/ferretdb/all-in-one/ferretdb.sh /etc/service/ferretdb/run
COPY --from=all-in-one-build /src/build/ferretdb/all-in-one/postgresql.sh /etc/service/postgresql/run
COPY --from=all-in-one-build /src/build/ferretdb/all-in-one/entrypoint.sh /entrypoint.sh
# Use moving development image during development.
# FROM ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb AS eval-dev
RUN --mount=type=cache,sharing=locked,target=/var/cache/apt <<EOF
set -ex
apt update
apt install -y curl runit sqlite3
apt install -y curl supervisor
curl -L https://pgp.mongodb.com/server-7.0.asc | apt-key add -
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/debian bookworm/mongodb-org/7.0 main" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list
apt update
apt install -y mongodb-mongosh
mkdir /tmp/cover
EOF
ENV FERRETDB_POSTGRESQL_URL=postgres://username:password@127.0.0.1:5432/ferretdb
ENV POSTGRES_USER=username
ENV POSTGRES_PASSWORD=password
ENV POSTGRES_DB=ferretdb
COPY --from=eval-dev-build /src/bin/ferretdb /usr/local/bin/ferretdb
STOPSIGNAL SIGHUP
ENTRYPOINT ["/entrypoint.sh"]
COPY --from=eval-dev-build /src/build/ferretdb/evaluation/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
COPY --from=eval-dev-build /src/build/ferretdb/evaluation/entrypoint.sh /usr/local/bin/entrypoint.sh
# all-in-one hacks stop there
ENTRYPOINT ["entrypoint.sh"]
HEALTHCHECK --interval=1m --timeout=5s --retries=1 --start-period=30s --start-interval=5s \
CMD ["/ferretdb", "ping"]
CMD ["/usr/local/bin/ferretdb", "ping"]
WORKDIR /
VOLUME /state
EXPOSE 27017 27018 8088
ENV GOCOVERDIR=/tmp/cover
ENV GORACE=halt_on_error=1,history_size=2
# don't forget to update documentation if you change defaults
ENV FERRETDB_LISTEN_ADDR=:27017
# ENV FERRETDB_LISTEN_TLS=:27018
ENV FERRETDB_DEBUG_ADDR=:8088
ENV FERRETDB_STATE_DIR=/state
ENV FERRETDB_SQLITE_URL=file:/state/
ARG LABEL_VERSION
ARG LABEL_COMMIT
# TODO https://github.com/FerretDB/FerretDB/issues/2212
LABEL org.opencontainers.image.description="A truly Open Source MongoDB alternative"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.description="A truly Open Source MongoDB alternative (evaluation development image)"
LABEL org.opencontainers.image.revision="${LABEL_COMMIT}"
LABEL org.opencontainers.image.source="https://github.com/FerretDB/FerretDB"
LABEL org.opencontainers.image.title="FerretDB"
LABEL org.opencontainers.image.title="FerretDB (evaluation development image)"
LABEL org.opencontainers.image.url="https://www.ferretdb.com/"
LABEL org.opencontainers.image.vendor="FerretDB Inc."
LABEL org.opencontainers.image.version="${LABEL_VERSION}"
+136
View File
@@ -0,0 +1,136 @@
# syntax=docker/dockerfile:1
# evaluation image (formerly "all-in-one") with production release (`ferretdb-eval` image)
# While we already know commit and version from commit.txt and version.txt inside image,
# it is not possible to use them in LABELs for the final image.
# We need to pass them as build arguments.
# Defining ARGs there makes them global.
ARG LABEL_VERSION
ARG LABEL_COMMIT
# prepare stage
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM --platform=$BUILDPLATFORM golang:1.25.4-bookworm AS eval-prepare
# use a single directory for all Go caches to simplify RUN --mount commands below
ENV GOPATH=/cache/gopath
ENV GOCACHE=/cache/gocache
ENV GOMODCACHE=/cache/gomodcache
# remove ",direct"
ENV GOPROXY=https://proxy.golang.org
COPY go.mod go.sum /src/
WORKDIR /src
RUN --mount=type=cache,target=/cache <<EOF
set -ex
go mod download
go mod verify
EOF
# build stage
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM golang:1.25.4-bookworm AS eval-build
ARG TARGETARCH
ARG LABEL_VERSION
ARG LABEL_COMMIT
RUN test -n "$LABEL_VERSION"
RUN test -n "$LABEL_COMMIT"
# use the same directories for Go caches as above
ENV GOPATH=/cache/gopath
ENV GOCACHE=/cache/gocache
ENV GOMODCACHE=/cache/gomodcache
# modules are already downloaded
ENV GOPROXY=off
# see .dockerignore
WORKDIR /src
COPY . .
# to add a dependency
COPY --from=eval-prepare /src/go.mod /src/go.sum /src/
RUN --mount=type=cache,target=/cache <<EOF
set -ex
git status
# Do not raise without providing separate builds with those values
# because higher versions are problematic for some virtualization platforms and older hardware.
export GOAMD64=v1
export GOARM64=v8.0
export CGO_ENABLED=0
go env
# Trim paths mostly to check that building with `-trimpath` is supported.
# check if stdlib was cached
go install -v -trimpath std
go build -v -trimpath -o=bin/ferretdb ./cmd/ferretdb
go version -m bin/ferretdb
bin/ferretdb --version
EOF
# final stage
# Use production image and full tag close to the release.
FROM ghcr.io/ferretdb/postgres-documentdb:17-0.107.0-ferretdb-2.7.0 AS eval
# Use moving development image during development.
# FROM ghcr.io/ferretdb/postgres-documentdb-dev:17-ferretdb AS eval
RUN --mount=type=cache,sharing=locked,target=/var/cache/apt <<EOF
apt install -y curl supervisor
curl -L https://pgp.mongodb.com/server-7.0.asc | apt-key add -
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/debian bookworm/mongodb-org/7.0 main" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list
apt update
apt install -y mongodb-mongosh
EOF
COPY --from=eval-build /src/bin/ferretdb /usr/local/bin/ferretdb
COPY --from=eval-build /src/build/ferretdb/evaluation/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
COPY --from=eval-build /src/build/ferretdb/evaluation/entrypoint.sh /usr/local/bin/entrypoint.sh
ENTRYPOINT ["entrypoint.sh"]
HEALTHCHECK --interval=1m --timeout=5s --retries=1 --start-period=30s --start-interval=5s \
CMD ["/usr/local/bin/ferretdb", "ping"]
VOLUME /state
EXPOSE 27017 27018 8088
# don't forget to update documentation if you change defaults
ENV FERRETDB_LISTEN_ADDR=:27017
# ENV FERRETDB_LISTEN_TLS=:27018
ENV FERRETDB_DEBUG_ADDR=:8088
ENV FERRETDB_STATE_DIR=/state
ARG LABEL_VERSION
ARG LABEL_COMMIT
# TODO https://github.com/FerretDB/FerretDB/issues/2212
LABEL org.opencontainers.image.description="A truly Open Source MongoDB alternative (evaluation image)"
LABEL org.opencontainers.image.revision="${LABEL_COMMIT}"
LABEL org.opencontainers.image.source="https://github.com/FerretDB/FerretDB"
LABEL org.opencontainers.image.title="FerretDB (evaluation image)"
LABEL org.opencontainers.image.url="https://www.ferretdb.com/"
LABEL org.opencontainers.image.vendor="FerretDB Inc."
LABEL org.opencontainers.image.version="${LABEL_VERSION}"
+20
View File
@@ -0,0 +1,20 @@
#!/bin/bash
set -e
if [ -z "$POSTGRES_PASSWORD" ]; then
echo "Error: POSTGRES_PASSWORD must be set. See https://docs.ferretdb.io/installation/evaluation/"
exit 1
fi
if [ "${POSTGRES_DB:-postgres}" != "postgres" ]; then
echo "Error: POSTGRES_DB must be set to 'postgres' or unset. See https://docs.ferretdb.io/installation/evaluation/"
exit 1
fi
# prevent unset POSTGRES_DB using the value of POSTGRES_USER, see https://hub.docker.com/_/postgres
export POSTGRES_DB="postgres"
export FERRETDB_POSTGRESQL_URL="postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}"
exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
@@ -0,0 +1,24 @@
; https://supervisord.org/configuration.html#supervisord-section-values
[supervisord]
nodaemon=true
loglevel=info
logfile=/dev/stderr
logfile_maxbytes=0
; https://supervisord.org/configuration.html#program-x-section-values
[program:postgres]
command=/usr/local/bin/docker-entrypoint.sh postgres
; lower priority postgres starts before ferretdb (default priority 999)
priority=1
; fast shutdown https://www.postgresql.org/docs/current/server-shutdown.html
stopsignal=INT
stdout_logfile=/dev/stderr
stdout_logfile_maxbytes=0
redirect_stderr=true
; https://supervisord.org/configuration.html#program-x-section-values
[program:ferretdb]
command=/usr/local/bin/ferretdb
stdout_logfile=/dev/stderr
stdout_logfile_maxbytes=0
redirect_stderr=true
+10 -13
View File
@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
# for production releases (`ferret` image)
# for production releases (`ferretdb` image)
# While we already know commit and version from commit.txt and version.txt inside image,
# it is not possible to use them in LABELs for the final image.
@@ -12,9 +12,10 @@ ARG LABEL_COMMIT
# prepare stage
FROM --platform=$BUILDPLATFORM golang:1.24.3 AS production-prepare
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM --platform=$BUILDPLATFORM golang:1.25.4-bookworm AS production-prepare
# use a single directory for all Go caches to simpliy RUN --mount commands below
# use a single directory for all Go caches to simplify RUN --mount commands below
ENV GOPATH=/cache/gopath
ENV GOCACHE=/cache/gocache
ENV GOMODCACHE=/cache/gomodcache
@@ -36,10 +37,10 @@ EOF
# build stage
FROM golang:1.24.3 AS production-build
# TODO https://github.com/FerretDB/FerretDB/issues/5449
FROM golang:1.25.4-bookworm AS production-build
ARG TARGETARCH
ARG TARGETVARIANT
ARG LABEL_VERSION
ARG LABEL_COMMIT
@@ -66,12 +67,10 @@ set -ex
git status
# Do not raise it without providing a separate v1 build
# because v2+ is problematic for some virtualization platforms and older hardware.
# Do not raise without providing separate builds with those values
# because higher versions are problematic for some virtualization platforms and older hardware.
export GOAMD64=v1
# Set GOARM explicitly due to https://github.com/docker-library/golang/issues/494.
export GOARM=${TARGETVARIANT#v}
export GOARM64=v8.0
export CGO_ENABLED=0
@@ -108,7 +107,7 @@ USER ferretdb:ferretdb
COPY --from=production-build /src/bin/ferretdb /ferretdb
COPY --from=production-build --chown=ferretdb:ferretdb /state /state
ENTRYPOINT ["/ferretdb"]
ENTRYPOINT [ "/ferretdb" ]
HEALTHCHECK --interval=1m --timeout=5s --retries=1 --start-period=30s --start-interval=5s \
CMD ["/ferretdb", "ping"]
@@ -122,14 +121,12 @@ ENV FERRETDB_LISTEN_ADDR=:27017
# ENV FERRETDB_LISTEN_TLS=:27018
ENV FERRETDB_DEBUG_ADDR=:8088
ENV FERRETDB_STATE_DIR=/state
ENV FERRETDB_SQLITE_URL=file:/state/
ARG LABEL_VERSION
ARG LABEL_COMMIT
# TODO https://github.com/FerretDB/FerretDB/issues/2212
LABEL org.opencontainers.image.description="A truly Open Source MongoDB alternative"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.revision="${LABEL_COMMIT}"
LABEL org.opencontainers.image.source="https://github.com/FerretDB/FerretDB"
LABEL org.opencontainers.image.title="FerretDB"
-26
View File
@@ -1,26 +0,0 @@
// Copy to test.js and run with `task testjs`.
(function () {
"use strict";
const coll = db.test;
coll.drop();
const init = [
{ _id: "double", v: 42.13 },
{ _id: "double-whole", v: 42.0 },
{ _id: "double-zero", v: 0.0 },
];
coll.insertMany(init);
const query = { v: { $gt: 42.0 } };
const expected = [{ _id: "double", v: 42.13 }];
const actual = coll.find(query).toArray();
assert.eq(expected, actual);
print("test.js passed!");
})();
+21
View File
@@ -3,5 +3,26 @@
(function () {
"use strict";
// Update the following example with your test.
const coll = db.test;
coll.drop();
const init = [
{ _id: "double", v: 42.13 },
{ _id: "double-whole", v: 42.0 },
{ _id: "double-zero", v: 0.0 },
];
coll.insertMany(init);
const query = { v: { $gt: 42.0 } };
const expected = [{ _id: "double", v: 42.13 }];
const actual = coll.find(query).toArray();
assert.eq(expected, actual);
print("test.js passed!");
})();
+8
View File
@@ -0,0 +1,8 @@
{
"mcpServers": {
"FerretDB": {
"type": "remote",
"url": "http://127.0.0.1:8081/mcp"
}
}
}
+16
View File
@@ -0,0 +1,16 @@
setParameter:
enableTestCommands: 1 # for sleep
net:
port: 47017 # matches host's port for replica set to work
bindIpAll: true
tls:
mode: disabled
security:
authorization: enabled
keyFile: /etc/mongod_keyfile.txt
replication:
oplogSizeMB: 512
replSetName: rs0
+4 -5
View File
@@ -1,12 +1,11 @@
setParameter:
enableTestCommands: 1 # for sleep
net:
port: 47017 # matches host's port for replica set to work
port: 37017 # matches host's port for replica set to work
bindIpAll: true
tls:
mode: disabled
security:
authorization: disabled
replication:
oplogSizeMB: 512
replSetName: rs0
-16
View File
@@ -1,16 +0,0 @@
net:
port: 47018 # matches host's port for replica set to work
bindIpAll: true
tls:
mode: requireTLS
certificateKeyFile: /etc/certs/server.pem
CAFile: /etc/certs/rootCA-cert.pem
clusterFile: /etc/certs/client.pem
security:
authorization: enabled # that also enables authentication
clusterAuthMode: x509
replication:
oplogSizeMB: 512
replSetName: rs0
+17
View File
@@ -0,0 +1,17 @@
-- Please do not merge changes in this file.
-- See https://pgtap.org/documentation.html#usingpgtap.
-- Start transaction and plan the tests.
BEGIN;
SELECT plan(1); -- number of test assertions
SET documentdb_core.bsonUseEJson = on;
-- Update the following example with your test.
SELECT pass();
-- Finish the tests and clean up.
SELECT * FROM finish();
ROLLBACK;
+2
View File
@@ -15,3 +15,5 @@ scan:
- .env
- .envrc
- build/certs/*.pem
- internal/dataapi/api/openapi.json
- internal/dataapi/api/openapi-original.json
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2021 FerretDB Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package version
import "runtime"
const (
// DocumentDB is a version of DocumentDB this version of FerretDB is compatible with.
DocumentDB = "0.107.0 gitref: HEAD sha:e63835403d buildId:0"
// DocumentDBURL points to the release page of the DocumentDB version above.
DocumentDBURL = "https://github.com/FerretDB/documentdb/releases/tag/v0.107.0-ferretdb-2.7.0"
)
// DocumentDBSafeToUpdate represents versions of DocumentDB that FerretDB can update.
var DocumentDBSafeToUpdate = []string{
"0.102.0 gitref: HEAD sha:80462f5 buildId:0", // v2.1.0
"0.103.0 gitref: HEAD sha:7514232 buildId:0", // v2.2.0
"0.104.0 gitref: HEAD sha:2045d0e buildId:0", // v2.3.0, v2.3.1
"0.105.0 gitref: HEAD sha:8453d93b buildId:0", // v2.4.0
"0.106.0 gitref: HEAD sha:beb9d25d98 buildId:0", // v2.5.0
// FerretDB v2.6.0 wasn't released
}
// PostgreSQLTest is a version of PostgreSQL used by tests.
var PostgreSQLTest string
func init() {
arch := "x86_64-pc-linux-gnu"
if runtime.GOARCH == "arm64" {
arch = "aarch64-unknown-linux-gnu"
}
PostgreSQLTest = "PostgreSQL 17.6 (Debian 17.6-2.pgdg12+1) on " + arch + ", " +
"compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit"
}
+1 -1
View File
@@ -24,7 +24,7 @@ import (
"strings"
"sync"
"github.com/FerretDB/FerretDB/internal/util/must"
"github.com/FerretDB/FerretDB/v2/internal/util/must"
)
// runGit runs `git` with given arguments and returns stdout.
+1 -1
View File
@@ -1 +1 @@
7.0.42
v7.0.77
+200 -92
View File
@@ -14,7 +14,7 @@
// Package version provides information about FerretDB version and build configuration.
//
// # Required files
// # Extra files
//
// The following generated text files may be present in this (`build/version`) directory during building:
// - version.txt (required) contains information about the FerretDB version in a format
@@ -25,46 +25,37 @@
//
// # Go build tags
//
// The following Go build tags (also known as build constraints) affect all builds of FerretDB,
// including embedded usage:
// The following Go build tags (also known as build constraints) affect builds of FerretDB:
//
// ferretdb_debug - enables debug build (see below; implied by builds with race detector)
// ferretdb_hana - enables Hana backend (alpha)
// ferretdb_no_postgresql - disables PostgreSQL backend
// ferretdb_no_sqlite - disables SQLite backend
// ferretdb_dev - enables development build (see below; implied by builds with race detector)
//
// # Debug builds
// # Development builds
//
// Debug builds of FerretDB behave differently in a few aspects:
// - Some values that are normally randomized are fixed or less randomized to make debugging easier.
// - Some internal errors cause crashes instead of being handled more gracefully.
// - Stack traces are collected more liberally.
// - Metrics are written to stderr on exit.
// - The default logging level is set to debug.
//
// Debug builds are orthogonal to production releases, development releases, and local/host builds.
// For example, the host build could be made non-debug, and the production release, in theory, could be a debug build.
//
// Currently, our production releases are non-debug builds,
// and all other builds and releases (development releases, all-in-one releases, local builds, etc.) are debug builds.
// Development builds of FerretDB behave differently in a few aspects:
// - they are significantly slower;
// - some values that are normally randomized are fixed or less randomized to make debugging easier;
// - some internal errors cause crashes instead of being handled more gracefully;
// - stack traces are collected more liberally;
// - metrics are written to stderr on exit;
// - the default logging level is set to debug.
package version
import (
"embed"
"fmt"
"regexp"
"runtime/debug"
"runtime"
runtimedebug "runtime/debug"
"strconv"
"strings"
"github.com/FerretDB/FerretDB/internal/types"
"github.com/FerretDB/FerretDB/internal/util/debugbuild"
"github.com/FerretDB/FerretDB/internal/util/must"
"github.com/FerretDB/FerretDB/v2/internal/util/devbuild"
"github.com/FerretDB/FerretDB/v2/internal/util/must"
)
// Each pattern in a //go:embed line must match at least one file or non-empty directory,
// but most files are generated and are not present when embeddable FerretDB package is used.
// As a workaround, mongodb.txt is always present.
// but most files are generated and may be absent (for example, when embeddable package is being used).
// As a workaround, mongodb.txt is always present and stored in the repository / Go module.
//go:generate go run ./generate.go
@@ -80,14 +71,14 @@ type Info struct {
Branch string
Dirty bool
Package string
DebugBuild bool
BuildEnvironment *types.Document
DevBuild bool
BuildEnvironment map[string]string
// MongoDBVersion is fake MongoDB version for clients that check major.minor to adjust their behavior.
MongoDBVersion string
// MongoDBVersionArray is MongoDBVersion, but as an array.
MongoDBVersionArray *types.Array
MongoDBVersionArray [4]int32
}
// info singleton instance set by init().
@@ -96,90 +87,207 @@ var info *Info
// unknown is a placeholder for unknown version, commit, and branch values.
const unknown = "unknown"
// FerretDB module path from go.mod.
const ferretdbModule = "github.com/FerretDB/FerretDB/v2"
// semVerTag is a https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string,
// but with a leading `v`.
//
//nolint:lll // for readability
var semVerTag = regexp.MustCompile(`^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
// Get returns current build's info.
//
// It returns a shared instance without any synchronization.
// If caller needs to modify the instance, it should make sure there is no concurrent accesses.
func Get() *Info {
return info
}
func init() {
versionRe := regexp.MustCompile(`^([0-9]+)\.([0-9]+)\.([0-9]+)$`)
// initFromFiles initializes info from txt files (that might be absent).
// All info fields are set to non-empty values, but some of them may be unknown.
func initFromFiles() {
mongodbTxt := strings.TrimSpace(string(must.NotFail(gen.ReadFile("mongodb.txt"))))
parts := versionRe.FindStringSubmatch(strings.TrimSpace(string(must.NotFail(gen.ReadFile("mongodb.txt")))))
if len(parts) != 4 {
match := semVerTag.FindStringSubmatch(mongodbTxt)
if match == nil || len(match) != semVerTag.NumSubexp()+1 {
panic("invalid mongodb.txt")
}
major := must.NotFail(strconv.ParseInt(parts[1], 10, 32))
minor := must.NotFail(strconv.ParseInt(parts[2], 10, 32))
patch := must.NotFail(strconv.ParseInt(parts[3], 10, 32))
major := must.NotFail(strconv.ParseInt(match[semVerTag.SubexpIndex("major")], 10, 32))
minor := must.NotFail(strconv.ParseInt(match[semVerTag.SubexpIndex("minor")], 10, 32))
patch := must.NotFail(strconv.ParseInt(match[semVerTag.SubexpIndex("patch")], 10, 32))
mongoDBVersion := fmt.Sprintf("%d.%d.%d", major, minor, patch)
mongoDBVersionArray := must.NotFail(types.NewArray(int32(major), int32(minor), int32(patch), int32(0)))
mongoDBVersionArray := [...]int32{int32(major), int32(minor), int32(patch), int32(0)}
info = &Info{
Version: unknown,
Commit: unknown,
Branch: unknown,
Dirty: false,
Package: unknown,
DebugBuild: debugbuild.Enabled,
BuildEnvironment: must.NotFail(types.NewDocument()),
Version: unknown,
Commit: unknown,
Branch: unknown,
Dirty: false,
Package: unknown,
DevBuild: devbuild.Enabled,
BuildEnvironment: map[string]string{
"go.runtime": runtime.Version(),
},
MongoDBVersion: mongoDBVersion,
MongoDBVersionArray: mongoDBVersionArray,
}
// do not expose extra information when embeddable FerretDB package is used
// (and some of it is most likely absent anyway)
buildInfo, ok := debug.ReadBuildInfo()
if !ok {
return
}
// TODO https://github.com/FerretDB/FerretDB/issues/4001
if buildInfo.Main.Path != "github.com/FerretDB/FerretDB" {
return
}
// Those files are present in two cases:
// 1. When we are working in the FerretDB repo (building binaries, running integration tests, etc),
// those files are generated by `task gen-version`.
// 2. When someone builds the FerretDB binary as described in the package description.
for f, sp := range map[string]*string{
"version.txt": &info.Version,
"commit.txt": &info.Commit,
"branch.txt": &info.Branch,
"package.txt": &info.Package,
} {
if b, _ := gen.ReadFile(f); len(b) > 0 {
*sp = strings.TrimSpace(string(b))
}
}
if !strings.HasPrefix(info.Version, "v") {
msg := "Invalid build/version/version.txt file content. Please run `bin/task gen-version`.\n"
msg += "Alternatively, create this file manually with a content similar to\n"
msg += "the output of `git describe`: `v<major>.<minor>.<patch>`.\n"
msg += "See https://pkg.go.dev/github.com/FerretDB/FerretDB/build/version"
panic(msg)
}
info.BuildEnvironment.Set("go.version", buildInfo.GoVersion)
for _, s := range buildInfo.Settings {
switch s.Key {
case "vcs.revision":
if s.Value != info.Commit {
// for non-official builds
if info.Commit == unknown {
info.Commit = s.Value
continue
}
panic(fmt.Sprintf("commit.txt value %q != vcs.revision value %q\n"+
"Please run `bin/task gen-version`", info.Commit, s.Value,
))
}
case "vcs.modified":
info.Dirty = must.NotFail(strconv.ParseBool(s.Value))
default:
info.BuildEnvironment.Set(s.Key, s.Value)
b, _ := gen.ReadFile(f)
if s := strings.TrimSpace(string(b)); s != "" {
*sp = s
}
}
}
// readBuildInfo returns FerretDB version and commit from the build info.
// It also updates info.BuildEnvironment and info.Dirty when it is sure we are building FerretDB,
// and not something that uses FerretDB.
func readBuildInfo() (version, commit string) {
// in theory, someone could use embeddable package in a Go program that is built without modules
buildInfo, ok := runtimedebug.ReadBuildInfo()
if !ok {
return
}
info.BuildEnvironment["go.version"] = buildInfo.GoVersion
// What buildInfo contains depends on what is being built and how.
// There are at least five cases:
//
// 1. Builds and unit tests in the FerretDB repo (task build-host, task test-unit, etc).
// Version is not set even for tags due to https://github.com/golang/go/issues/72877.
// Path: "github.com/FerretDB/FerretDB/v2/cmd/ferretdb"
// (or "github.com/FerretDB/FerretDB/v2/build/version.test", etc)
// Main.Path: "github.com/FerretDB/FerretDB/v2"
// Main.Version: "(devel)"
//
// 2. Builds with known module version (go install github.com/FerretDB/FerretDB/v2/cmd/ferretdb@v2.0.0).
// Path: "github.com/FerretDB/FerretDB/v2/cmd/ferretdb"
// Main.Path: "github.com/FerretDB/FerretDB/v2"
// Main.Version: "v2.0.0"
//
// 3. Ad-hoc builds (go run main.go readyz.go --dev-version) due to https://github.com/golang/go/issues/51279.
// Path: "command-line-arguments"
// Main.Path: ""
// Main.Version: ""
// Deps.Path: "github.com/FerretDB/FerretDB/v2"
// Deps.Version: "(devel)"
//
// 4. Integration tests (both in the test and in the command handler).
// Path: "github.com/FerretDB/FerretDB/v2/integration.test"
// Main.Path: "github.com/FerretDB/FerretDB/v2/integration"
// Main.Version: "(devel)"
// Deps: null
//
// 5. Embeddable package.
// Path: ???
// Main.Path: ???
// Deps.Path: "github.com/FerretDB/FerretDB/v2"
// Deps.Version: "v2.0.0"
// With replace directive in go.mod:
// Deps.Replace.Path: "../FerretDB"
// Deps.Replace.Version: "(devel)"
//
// In short, FerretDB version could be derived from the module version only when module version could be known,
// which is not often the case. And there are bugs in the Go toolchain that prevent version from being set
// even when it is known.
// cases 1 and 2
if buildInfo.Main.Path == ferretdbModule {
version = buildInfo.Main.Version
// in the case 1 even for tags due to https://github.com/golang/go/issues/72877
if version == "(devel)" {
version = ""
}
for _, s := range buildInfo.Settings {
if v := s.Value; v != "" {
info.BuildEnvironment[s.Key] = v
}
// both are present only in the case 1
switch s.Key {
case "vcs.revision":
commit = s.Value
case "vcs.modified":
info.Dirty = must.NotFail(strconv.ParseBool(s.Value))
}
}
return
}
// cases 3, 4, and 5
for _, dep := range buildInfo.Deps {
if dep.Path != ferretdbModule {
continue
}
version = dep.Version
if dep.Replace != nil {
version = dep.Replace.Version
}
// cases 3 and 4
if version == "(devel)" {
version = ""
}
// vcs.revision and other settings refer to the repository that uses FerretDB, not FerretDB itself.
// We don't want git commit hashes or build configurations of other repos.
break
}
return
}
func init() {
initFromFiles()
version, commit := readBuildInfo()
if info.Version == unknown && version != "" {
info.Version = version
}
if info.Commit == unknown && commit != "" {
info.Commit = commit
}
if info.Version != unknown {
if match := semVerTag.FindStringSubmatch(info.Version); match == nil || len(match) != semVerTag.NumSubexp()+1 {
msg := fmt.Sprintf("info.Version: %q, version: %q\n", info.Version, version)
msg += "Invalid build/version/version.txt file content. Please run `bin/task gen-version`.\n"
msg += "Alternatively, create this file manually with a content similar to\n"
msg += "the output of `git describe`: `v<major>.<minor>.<patch>`.\n"
msg += "See https://pkg.go.dev/github.com/FerretDB/FerretDB/v2/build/version"
panic(msg)
}
}
// This does not work for git submodules.
// Investigate and create an issue.
//
// if info.Commit != unknown {
// if info.Commit != commit && commit != "" {
// msg := fmt.Sprintf("info.Commit: %q, commit: %q\n", info.Commit, commit)
// msg += "Invalid build/version/commit.txt file content. Please run `bin/task gen-version`."
// panic(msg)
// }
// }
}
+13 -9
View File
@@ -15,20 +15,24 @@
package version
import (
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/FerretDB/FerretDB/internal/types"
"github.com/FerretDB/FerretDB/internal/util/must"
"github.com/FerretDB/FerretDB/internal/util/testutil"
)
func TestGet(t *testing.T) {
v := Get()
func TestCase1(t *testing.T) {
assert.Regexp(t, semVerTag, info.Version)
assert.Regexp(t, `^[0-9a-f]{40}$`, info.Commit)
assert.NotEmpty(t, info.Branch)
assert.NotEqual(t, unknown, info.Branch)
assert.NotEmpty(t, info.Package)
// package is unknown on CI for short tests where package.txt is not created
assert.NotEmpty(t, v.Version)
assert.Equal(t, "7.0.77", info.MongoDBVersion)
assert.Equal(t, [...]int32{int32(7), int32(0), int32(77), int32(0)}, info.MongoDBVersionArray)
assert.Equal(t, "7.0.42", v.MongoDBVersion)
testutil.AssertEqual(t, must.NotFail(types.NewArray(int32(7), int32(0), int32(42), int32(0))), v.MongoDBVersionArray)
assert.Equal(t, runtime.Version(), info.BuildEnvironment["go.version"])
assert.Equal(t, runtime.Version(), info.BuildEnvironment["go.runtime"])
assert.Empty(t, info.BuildEnvironment["vcs.revision"]) // not set for unit tests
}
+1
View File
@@ -0,0 +1 @@
CREATE EXTENSION IF NOT EXISTS documentdb CASCADE;
+20 -245
View File
@@ -23,8 +23,6 @@ import (
"html/template"
"io"
"log/slog"
"net"
"net/url"
"os"
"os/exec"
"path/filepath"
@@ -34,16 +32,10 @@ import (
"time"
"github.com/alecthomas/kong"
"github.com/prometheus/client_golang/prometheus"
"github.com/FerretDB/FerretDB/build/version"
mysqlpool "github.com/FerretDB/FerretDB/internal/backends/mysql/metadata/pool"
"github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata/pool"
"github.com/FerretDB/FerretDB/internal/util/ctxutil"
"github.com/FerretDB/FerretDB/internal/util/debug"
"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
"github.com/FerretDB/FerretDB/internal/util/logging"
"github.com/FerretDB/FerretDB/internal/util/state"
"github.com/FerretDB/FerretDB/v2/build/version"
"github.com/FerretDB/FerretDB/v2/internal/util/ctxutil"
"github.com/FerretDB/FerretDB/v2/internal/util/logging"
)
var (
@@ -57,225 +49,6 @@ var (
// versionFile contains version information with leading v.
const versionFile = "build/version/version.txt"
// waitForPort waits for the given port to be available until ctx is canceled.
func waitForPort(ctx context.Context, logger *slog.Logger, port uint16) error {
addr := fmt.Sprintf("127.0.0.1:%d", port)
logger.InfoContext(ctx, fmt.Sprintf("Waiting for %s to be up", addr))
var retry int64
for ctx.Err() == nil {
conn, err := net.Dial("tcp", addr)
if err == nil {
conn.Close()
return nil
}
logger.InfoContext(ctx, fmt.Sprintf("%s: %s", addr, err))
retry++
ctxutil.SleepWithJitter(ctx, time.Second, retry)
}
return fmt.Errorf("failed to connect to %s", addr)
}
// setupAnyPostgres configures given PostgreSQL.
func setupAnyPostgres(ctx context.Context, logger *slog.Logger, uri string) error {
u, err := url.Parse(uri)
if err != nil {
return err
}
port, err := strconv.ParseUint(u.Port(), 10, 16)
if err != nil {
return err
}
if u.User == nil {
return lazyerrors.New("No username specified")
}
if err = waitForPort(ctx, logger, uint16(port)); err != nil {
return err
}
sp, err := state.NewProvider("")
if err != nil {
return err
}
p, err := pool.New(uri, logger, sp)
if err != nil {
return err
}
defer p.Close()
username := u.User.Username()
password, _ := u.User.Password()
var retry int64
for ctx.Err() == nil {
if _, err = p.Get(username, password); err == nil {
break
}
logger.InfoContext(ctx, fmt.Sprintf("%s: %s", uri, err))
retry++
ctxutil.SleepWithJitter(ctx, time.Second, retry)
}
if ctx.Err() != nil {
return ctx.Err()
}
return nil
}
// setupPostgres configures `postgres` container.
func setupPostgres(ctx context.Context, logger *slog.Logger) error {
// user `username` must exist, but password may be any, even empty
return setupAnyPostgres(ctx, logging.WithName(logger, "postgres"), "postgres://username@127.0.0.1:5432/ferretdb")
}
// setupPostgresSecured configures `postgres_secured` container.
func setupPostgresSecured(ctx context.Context, logger *slog.Logger) error {
return setupAnyPostgres(ctx, logging.WithName(logger, "postgres_secured"), "postgres://username:password@127.0.0.1:5433/ferretdb") //nolint:lll // for readability
}
// setupMySQL configures `mysql` container.
func setupMySQL(ctx context.Context, logger *slog.Logger) error {
uri := "mysql://root:password@127.0.0.1:3306/ferretdb"
sp, err := state.NewProvider("")
if err != nil {
return err
}
if err = waitForPort(ctx, logging.WithName(logger, "mysql"), 3306); err != nil {
return err
}
p, err := mysqlpool.New(uri, logger, sp)
if err != nil {
return err
}
defer p.Close()
var retry int64
for ctx.Err() == nil {
db, err := p.Get("root", "password")
if err == nil {
if _, err = db.ExecContext(ctx, "GRANT ALL PRIVILEGES ON *.* TO 'username'@'%';"); err != nil {
return lazyerrors.Error(err)
}
break
}
logger.InfoContext(ctx, fmt.Sprintf("%s: %s", uri, err))
retry++
ctxutil.SleepWithJitter(ctx, time.Second, retry)
}
if ctx.Err() != nil {
return ctx.Err()
}
return nil
}
// setupMongodb configures `mongodb` container.
func setupMongodb(ctx context.Context, logger *slog.Logger) error {
if err := waitForPort(ctx, logging.WithName(logger, "mongodb"), 47017); err != nil {
return err
}
eval := `'rs.initiate({_id: "rs0", members: [{_id: 0, host: "localhost:47017" }]})'`
args := []string{"compose", "exec", "-T", "mongodb", "mongosh", "--port=47017", "--eval", eval}
var buf bytes.Buffer
var retry int64
for ctx.Err() == nil {
buf.Reset()
err := runCommand("docker", args, &buf, logger)
if err == nil {
break
}
logger.InfoContext(ctx, fmt.Sprintf("%s:\n%s", err, buf.String()))
retry++
ctxutil.SleepWithJitter(ctx, time.Second, retry)
}
return ctx.Err()
}
// setupMongodbSecured configures `mongodb_secured` container.
func setupMongodbSecured(ctx context.Context, logger *slog.Logger) error {
if err := waitForPort(ctx, logging.WithName(logger, "mongodb_secured"), 47018); err != nil {
return err
}
eval := `'rs.initiate({_id: "rs0", members: [{_id: 0, host: "localhost:47018" }]})'`
shell := `mongodb://username:password@127.0.0.1:47018/?tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem` //nolint:lll // for readability
args := []string{"compose", "exec", "-T", "mongodb_secured", "mongosh", "--eval", eval, "--shell", shell}
var buf bytes.Buffer
var retry int64
for ctx.Err() == nil {
buf.Reset()
err := runCommand("docker", args, &buf, logger)
if err == nil {
break
}
logger.InfoContext(ctx, fmt.Sprintf("%s:\n%s", err, buf.String()))
retry++
ctxutil.SleepWithJitter(ctx, time.Second, retry)
}
return ctx.Err()
}
// setup runs all setup commands.
func setup(ctx context.Context, logger *slog.Logger) error {
h, err := debug.Listen(&debug.ListenOpts{
TCPAddr: "127.0.0.1:8089",
L: logging.WithName(logger, "debug"),
R: prometheus.DefaultRegisterer,
})
if err != nil {
return lazyerrors.Error(err)
}
go h.Serve(ctx)
for _, f := range []func(context.Context, *slog.Logger) error{
setupPostgres,
setupPostgresSecured,
setupMySQL,
setupMongodb,
setupMongodbSecured,
} {
if err := f(ctx, logger); err != nil {
return err
}
}
logger.InfoContext(ctx, "Done")
return nil
}
// runCommand runs command with given arguments.
func runCommand(command string, args []string, stdout io.Writer, logger *slog.Logger) error {
bin, err := exec.LookPath(command)
@@ -338,12 +111,12 @@ func printDiagnosticData(w io.Writer, setupError error, logger *slog.Logger) err
"GOOS": runtime.GOOS,
"GOARCH": runtime.GOARCH,
"Version": info.Version,
"Commit": info.Commit,
"Branch": info.Branch,
"Dirty": info.Dirty,
"Package": info.Package,
"DebugBuild": info.DebugBuild,
"Version": info.Version,
"Commit": info.Commit,
"Branch": info.Branch,
"Dirty": info.Dirty,
"Package": info.Package,
"DevBuild": info.DevBuild,
"GoVersion": runtime.Version(),
"GitVersion": strings.TrimSpace(gitVersion),
@@ -417,6 +190,7 @@ type TestsRunParams struct {
ShardTotal uint `help:"Total number of shards."`
Run string `help:"Run only tests matching the regexp."`
Skip string `help:"Skip tests matching the regexp."`
RawPrefix string `help:"Prefix for raw output files."`
Args []string `arg:"" help:"Other arguments and flags for 'go test'." passthrough:""`
}
@@ -434,13 +208,13 @@ var cli struct {
Shell struct {
Mkdir struct {
Paths []string `arg:"" name:"path" help:"Paths to create." type:"path"`
Paths []string `name:"path" arg:"" help:"Paths to create." type:"path"`
} `cmd:"" help:"Create directories if they do not already exist."`
Rmdir struct {
Paths []string `arg:"" name:"path" help:"Paths to remove." type:"path"`
Paths []string `name:"path" arg:"" help:"Paths to remove." type:"path"`
} `cmd:"" help:"Remove directories."`
Read struct {
Paths []string `arg:"" name:"path" help:"Paths to read." type:"path"`
Paths []string `name:"path" arg:"" help:"Paths to read." type:"path"`
} `cmd:"" help:"Read files."`
} `cmd:""`
@@ -457,10 +231,10 @@ var cli struct {
}
func main() {
kongCtx := kong.Parse(&cli)
kongCtx := kong.Parse(&cli, kong.DefaultEnvars("ENVTOOL"))
// https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
if t, _ := strconv.ParseBool(os.Getenv("CI")); t {
if t, _ := strconv.ParseBool(os.Getenv("RUNNER_DEBUG")); t {
cli.Debug = true
}
@@ -474,9 +248,10 @@ func main() {
Level: level,
RemoveTime: true,
RemoveSource: true,
// the logger handles messages from the `go test` command output
SkipChecks: true,
}
logging.Setup(opts, "")
logging.SetupDefault(opts, "")
logger := slog.Default()
cmd := kongCtx.Command()
@@ -486,7 +261,7 @@ func main() {
switch cmd {
case "setup":
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
err = setup(ctx, logger)
@@ -562,7 +337,7 @@ func main() {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
logger.Error("Failed to exit", logging.Error(exitErr))
logger.Error("Failed", logging.Error(exitErr))
os.Exit(exitErr.ExitCode())
}
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/FerretDB/FerretDB/internal/util/testutil"
"github.com/FerretDB/FerretDB/v2/internal/util/testutil"
)
func TestPrintDiagnosticData(t *testing.T) {
+1 -1
View File
@@ -7,7 +7,7 @@ Version: {{.Version}}
Commit: {{.Commit}}
Branch: {{.Branch}}
Dirty: {{.Dirty}}
Debug: {{.DebugBuild}}
Dev: {{.DevBuild}}
{{.GoVersion}}
{{.GitVersion}}
+3 -3
View File
@@ -28,7 +28,7 @@ import (
"slices"
"strings"
"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
"github.com/AlekSi/lazyerrors"
)
// copyFile copies a file from src to dst, overwriting dst if it exists.
@@ -68,7 +68,7 @@ func copyFile(src, dst string) error {
return nil
}
// fuzzGeneratedCorpus returns $GOCACHE/fuzz/github.com/FerretDB/FerretDB,
// fuzzGeneratedCorpus returns $GOCACHE/fuzz/github.com/FerretDB/FerretDB/v2,
// ensuring that this directory exists.
func fuzzGeneratedCorpus() (string, error) {
b, err := exec.Command("go", "env", "GOCACHE").Output()
@@ -76,7 +76,7 @@ func fuzzGeneratedCorpus() (string, error) {
return "", lazyerrors.Error(err)
}
path := filepath.Join(string(bytes.TrimSpace(b)), "fuzz", "github.com", "FerretDB", "FerretDB")
path := filepath.Join(string(bytes.TrimSpace(b)), "fuzz", "github.com", "FerretDB", "FerretDB", "v2")
if _, err = os.Stat(path); err != nil {
if errors.Is(err, fs.ErrNotExist) {
+182
View File
@@ -0,0 +1,182 @@
// Copyright 2021 FerretDB Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"fmt"
"log/slog"
"net"
"net/url"
"time"
"github.com/AlekSi/lazyerrors"
"github.com/FerretDB/wire/wirebson"
"github.com/jackc/pgx/v5"
"github.com/prometheus/client_golang/prometheus"
"github.com/FerretDB/FerretDB/v2/internal/documentdb"
"github.com/FerretDB/FerretDB/v2/internal/util/ctxutil"
"github.com/FerretDB/FerretDB/v2/internal/util/debug"
"github.com/FerretDB/FerretDB/v2/internal/util/logging"
"github.com/FerretDB/FerretDB/v2/internal/util/state"
)
// setupMongoDB configures MongoDB containers.
func setupMongoDB(ctx context.Context, logger *slog.Logger, uri, name string) error {
u, err := url.Parse(uri)
if err != nil {
return lazyerrors.Error(err)
}
_, port, err := net.SplitHostPort(u.Host)
if err != nil {
return lazyerrors.Error(err)
}
logger = logging.WithName(logger, name)
eval := fmt.Sprintf(`'rs.initiate({_id: "rs0", members: [{_id: 0, host: "localhost:%s" }]})'`, port)
args := []string{"compose", "exec", "-T", name, "mongosh", "--eval", eval, uri}
var buf bytes.Buffer
var attempt int64
for ctx.Err() == nil {
buf.Reset()
err := runCommand("docker", args, &buf, logger)
if err == nil {
break
}
logger.InfoContext(ctx, fmt.Sprintf("%s:\n%s", err, buf.String()))
attempt++
ctxutil.SleepWithJitter(ctx, time.Second, attempt)
}
return ctx.Err()
}
// setupUser waits for the given PostgreSQL URI to become available, and creates
// an admin user with fixed credentials `username:password` using DocumentDB API.
func setupUser(ctx context.Context, uri string, l *slog.Logger) error {
sp, err := state.NewProvider("")
if err != nil {
return lazyerrors.Error(err)
}
pool, err := documentdb.NewPool(uri, l, sp)
if err != nil {
return lazyerrors.Error(err)
}
defer pool.Close()
var retry int64
createUser := wirebson.MustDocument(
"createUser", "username",
"pwd", "password",
"roles", wirebson.MustArray(
wirebson.MustDocument(
"role", "clusterAdmin",
"db", "admin",
),
wirebson.MustDocument(
"role", "readWriteAnyDatabase",
"db", "admin",
),
),
)
var res wirebson.RawDocument
for ctx.Err() == nil {
err = pool.WithConn(func(conn *pgx.Conn) error {
res, err = documentdb.CreateUser(ctx, conn, l, createUser)
return err
})
if err == nil {
break
}
l.InfoContext(ctx, "Waiting for DocumentDB extension to be created")
retry++
ctxutil.SleepWithJitter(ctx, time.Second, retry)
}
if err != nil {
return lazyerrors.Error(err)
}
d, err := res.Decode()
if err != nil {
return lazyerrors.Error(err)
}
switch ok := d.Get("ok").(type) {
case float64:
if ok != float64(1) {
return lazyerrors.Errorf("Failed to create user: %s", d.LogMessage())
}
case int32:
// TODO https://github.com/FerretDB/FerretDB/issues/5313
if ok != int32(1) {
return lazyerrors.Errorf("Failed to create user: %s", d.LogMessage())
}
default:
panic("unexpected type")
}
l.InfoContext(ctx, "User created", slog.String("response", d.LogMessage()))
return nil
}
// setup runs all setup commands.
func setup(ctx context.Context, logger *slog.Logger) error {
h, err := debug.Listen(&debug.ListenOpts{
TCPAddr: "127.0.0.1:8089",
L: logging.WithName(logger, "debug"),
R: prometheus.DefaultRegisterer,
})
if err != nil {
return lazyerrors.Error(err)
}
go h.Run(ctx)
if err = setupMongoDB(ctx, logger, "mongodb://username:password@127.0.0.1:47017/", "mongodb-secure"); err != nil {
return lazyerrors.Error(err)
}
uri := "postgres://pg-user:pg-pass@127.0.0.1:5432/postgres"
if err = setupUser(ctx, uri, logging.WithName(logger, "postgres")); err != nil {
return lazyerrors.Error(err)
}
uri = "postgres://pg-user:pg-pass@127.0.0.1:5433/yugabyte"
if err = setupUser(ctx, uri, logging.WithName(logger, "yugabytedb")); err != nil {
return lazyerrors.Error(err)
}
logger.InfoContext(ctx, "Done")
return nil
}
+197 -79
View File
@@ -23,6 +23,7 @@ import (
"fmt"
"io"
"log/slog"
"maps"
"os"
"os/exec"
"slices"
@@ -30,20 +31,19 @@ import (
"strings"
"time"
"github.com/AlekSi/lazyerrors"
"go.opentelemetry.io/otel"
otelattribute "go.opentelemetry.io/otel/attribute"
otelcodes "go.opentelemetry.io/otel/codes"
oteltrace "go.opentelemetry.io/otel/trace"
"golang.org/x/exp/maps"
"github.com/FerretDB/FerretDB/internal/util/lazyerrors"
"github.com/FerretDB/FerretDB/internal/util/must"
"github.com/FerretDB/FerretDB/internal/util/observability"
"github.com/FerretDB/FerretDB/v2/internal/util/must"
"github.com/FerretDB/FerretDB/v2/internal/util/observability"
)
// testEvent represents a single even emitted by `go test -json`.
// testEvent represents a single event emitted by `go test -json` or `go build -json`.
//
// See https://pkg.go.dev/cmd/test2json#hdr-Output_Format.
// See `go doc test2json` and `go help buildjson`.
type testEvent struct {
Time time.Time `json:"Time"`
Action string `json:"Action"`
@@ -51,6 +51,8 @@ type testEvent struct {
Test string `json:"Test"`
Output string `json:"Output"`
ElapsedSeconds float64 `json:"Elapsed"`
ImportPath string `json:"ImportPath"`
}
// Elapsed returns an elapsed time.
@@ -69,15 +71,18 @@ type testResult struct {
// listTestFuncs returns a sorted slice of all top-level test functions (tests, benchmarks, examples, fuzz functions)
// matching given regular expression in the specified directory and subdirectories.
func listTestFuncs(dir, re string, logger *slog.Logger) ([]string, error) {
func listTestFuncs(ctx context.Context, dir, re string, logger *slog.Logger) ([]string, error) {
ctx, span := otel.Tracer("").Start(ctx, "envtool.listTestFuncs")
defer span.End()
var buf bytes.Buffer
cmd := exec.Command("go", "test", "-list="+re, "./...")
cmd := exec.CommandContext(ctx, "go", "test", "-list="+re, "./...")
cmd.Dir = dir
cmd.Stdout = &buf
cmd.Stderr = os.Stderr
logger.Info(fmt.Sprintf("Running %s", strings.Join(cmd.Args, " ")))
logger.InfoContext(ctx, "Running tests", slog.String("command", strings.Join(cmd.Args, " ")))
if err := cmd.Run(); err != nil {
return nil, lazyerrors.Error(err)
@@ -114,10 +119,7 @@ func listTestFuncs(dir, re string, logger *slog.Logger) ([]string, error) {
return nil, lazyerrors.Error(err)
}
res := maps.Keys(testFuncs)
slices.Sort(res)
return res, nil
return slices.Sorted(maps.Keys(testFuncs)), nil
}
// shardTestFuncs shards given top-level test functions.
@@ -155,7 +157,10 @@ func shardTestFuncs(index, total uint, testFuncs []string) ([]string, error) {
}
// testArgs handles `envtool tests run` arguments and returns a slice of `go test` arguments.
func testArgs(dir string, index, total uint, run, skip string, logger *slog.Logger) ([]string, uint, error) {
func testArgs(ctx context.Context, dir string, index, total uint, run, skip string, logger *slog.Logger) ([]string, uint, error) {
ctx, span := otel.Tracer("").Start(ctx, "envtool.testArgs")
defer span.End()
listRE := "."
if run != "" || skip != "" {
@@ -181,7 +186,7 @@ func testArgs(dir string, index, total uint, run, skip string, logger *slog.Logg
listRE = run
}
testFuncs, err := listTestFuncs(dir, listRE, logger)
testFuncs, err := listTestFuncs(ctx, dir, listRE, logger)
if err != nil {
return nil, 0, err
}
@@ -216,24 +221,111 @@ func resultKey(packageName, testName string) string {
return packageName + "." + testName
}
// runGoTest runs `go test` with given extra args.
func runGoTest(runCtx context.Context, args []string, total uint, times bool, logger *slog.Logger) error {
cmd := exec.CommandContext(runCtx, "go", append([]string{"test", "-json"}, args...)...)
// startGoTest starts `go test` with given arguments.
func startGoTest(ctx context.Context, args []string, raw string, l *slog.Logger) (*json.Decoder, func() error, error) {
var cleanups []func() error
logger.InfoContext(runCtx, fmt.Sprintf("Running %s", strings.Join(cmd.Args, " ")))
cleanup := func() error {
var err error
for _, f := range slices.Backward(cleanups) {
if e := f(); e != nil {
l.DebugContext(ctx, fmt.Sprintf("Cleanup error: %s", e))
if err == nil {
err = lazyerrors.Error(e)
}
}
}
return err
}
cmd := exec.CommandContext(ctx, "go", append([]string{"test", "-json"}, args...)...)
l.InfoContext(ctx, fmt.Sprintf("Running %s", strings.Join(cmd.Args, " ")))
cmd.Stderr = os.Stderr
p, err := cmd.StdoutPipe()
if err != nil {
return lazyerrors.Error(err)
if raw != "" {
f, e := os.Create(raw + "-stderr.txt.tmp")
if e != nil {
_ = cleanup()
return nil, nil, lazyerrors.Error(e)
}
cleanups = append(cleanups, f.Close)
cmd.Stderr = io.MultiWriter(f, os.Stderr)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
_ = cleanup()
return nil, nil, lazyerrors.Error(err)
}
d := json.NewDecoder(stdout)
if raw != "" {
f, e := os.Create(raw + "-stdout.json.tmp")
if e != nil {
_ = cleanup()
return nil, nil, lazyerrors.Error(e)
}
cleanups = append(cleanups, f.Close)
d = json.NewDecoder(io.TeeReader(stdout, f))
}
d.DisallowUnknownFields()
if err = cmd.Start(); err != nil {
return lazyerrors.Error(err)
_ = cleanup()
return nil, nil, lazyerrors.Error(err)
}
defer cmd.Cancel() //nolint:errcheck // safe to ignore
cleanups = append(cleanups, cmd.Wait)
return d, cleanup, nil
}
// runGoTestOpts represents [runGoTest] options.
//
//nolint:vet // for readability
type runGoTestOpts struct {
args []string
total uint
times bool
rawPrefix string
logger *slog.Logger
}
// runGoTest runs `go test` with given extra args.
func runGoTest(runCtx context.Context, opts *runGoTestOpts) (resErr error) {
totalTests := "?"
if opts.total > 0 {
totalTests = strconv.Itoa(int(opts.total))
}
runCtx, runSpan := otel.Tracer("").Start(runCtx, "envtool.runGoTest")
runSpan.SetAttributes(otelattribute.String("db.ferretdb.envtool.total_tests", totalTests))
defer runSpan.End()
d, cleanup, err := startGoTest(runCtx, opts.args, opts.rawPrefix, opts.logger)
if err != nil {
resErr = lazyerrors.Error(err)
return
}
defer func() {
if e := cleanup(); resErr == nil && e != nil {
resErr = lazyerrors.Error(e)
}
}()
runSpan.AddEvent("started")
// Keys are:
// - "package/name"
@@ -244,29 +336,29 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
results := make(map[string]*testResult, 300)
var done int
d := json.NewDecoder(p)
d.DisallowUnknownFields()
totalTests := "?"
if total > 0 {
totalTests = strconv.Itoa(int(total))
}
runCtx, runSpan := otel.Tracer("").Start(runCtx, "run")
runSpan.SetAttributes(otelattribute.String("db.ferretdb.envtool.total_tests", totalTests))
defer runSpan.End()
var firstEvent bool
for {
var event testEvent
if err = d.Decode(&event); err != nil {
if !errors.Is(err, io.EOF) {
return lazyerrors.Error(err)
resErr = lazyerrors.Error(err)
return
}
break
}
// skip build events for now
if event.ImportPath != "" {
continue
}
if !firstEvent {
runSpan.AddEvent("first event")
firstEvent = true
}
must.NotBeZero(event.Package)
key := resultKey(event.Package, event.Test)
@@ -285,9 +377,10 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
key = resultKey(event.Package, parentTest(event.Test))
parent := results[key]
// TODO https://github.com/FerretDB/FerretDB/issues/4465
if parent == nil {
panic(fmt.Sprintf("no parent test found: package=%q, test=%q, key=%q", event.Package, event.Test, key))
panic(fmt.Sprintf(
"no parent test found: package=%q, test=%q, key=%q", event.Package, event.Test, key,
))
}
parentCtx = parent.ctx
@@ -337,7 +430,7 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
// initial setup output or early panic
if event.Test == "" {
logger.InfoContext(runCtx, out)
opts.logger.InfoContext(runCtx, out)
continue
}
@@ -362,7 +455,7 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
testSpan.End()
if event.Test == "" {
logger.InfoContext(runCtx, strings.ToTitle(event.Action)+" "+event.Package)
opts.logger.InfoContext(runCtx, strings.ToTitle(event.Action)+" "+event.Package)
continue
}
@@ -373,7 +466,7 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
msg := strings.ToTitle(event.Action) + " " + event.Test
if times {
if opts.times {
msg += fmt.Sprintf(" (%.2fs", event.Time.Sub(res.cont).Seconds())
if res.run != res.cont {
@@ -392,26 +485,44 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
msg += fmt.Sprintf(" %d/%s", done, totalTests)
}
logOutput := opts.logger.ErrorContext
if event.Action == "skip" {
logOutput = opts.logger.WarnContext
}
if event.Action == "pass" {
logger.InfoContext(runCtx, msg)
continue
if !opts.logger.Enabled(runCtx, slog.LevelDebug) {
opts.logger.InfoContext(runCtx, msg)
continue
}
logOutput = opts.logger.DebugContext
}
msg += ":"
logger.WarnContext(runCtx, msg)
logOutput(runCtx, msg)
for _, l := range res.outputs {
logger.WarnContext(runCtx, l)
logOutput(runCtx, l)
}
logger.WarnContext(runCtx, "")
logOutput(runCtx, "")
default:
return lazyerrors.Errorf("unknown action %q", event.Action)
resErr = lazyerrors.Errorf("unknown action %q", event.Action)
return
}
}
var unfinished []string
logUnfinishedTests(runCtx, opts.logger, results)
return
}
// logUnfinishedTests logs tests that panicked or somehow unfinished, trying to provide the best possible output.
func logUnfinishedTests(runCtx context.Context, logger *slog.Logger, results map[string]*testResult) {
var tests []string
for t, res := range results {
switch res.lastAction {
@@ -419,45 +530,39 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
continue
}
unfinished = append(unfinished, t)
tests = append(tests, t)
}
if unfinished == nil {
return cmd.Wait()
if tests == nil {
return
}
slices.Sort(unfinished)
slices.Sort(tests)
logger.ErrorContext(runCtx, "")
logger.ErrorContext(runCtx, "Some tests did not finish:")
for _, t := range unfinished {
for _, t := range tests {
logger.ErrorContext(runCtx, fmt.Sprintf(" %s", t))
}
logger.ErrorContext(runCtx, "")
// On panic, the last event will not be "fail"; see https://github.com/golang/go/issues/38382.
// Try to provide the best possible output in that case.
var panicked string
for _, t := range unfinished {
if !slices.ContainsFunc(results[t].outputs, func(s string) bool {
for _, t := range tests {
if slices.ContainsFunc(results[t].outputs, func(s string) bool {
return strings.Contains(s, "panic: ")
}) {
continue
}
if panicked != "" {
panicked = t
break
}
panicked = t
}
for _, t := range unfinished {
// We can't always associate the panic with test; see https://github.com/golang/go/issues/38382.
// If we located the test, log only it. Otherwise, log all unfinished tests.
for _, t := range tests {
if panicked != "" && t != panicked {
continue
}
@@ -470,8 +575,6 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
logger.ErrorContext(runCtx, "")
}
return cmd.Wait()
}
// testsRun runs tests specified by the shard index and total or by the run regex
@@ -479,22 +582,17 @@ func runGoTest(runCtx context.Context, args []string, total uint, times bool, lo
func testsRun(ctx context.Context, params *TestsRunParams, logger *slog.Logger) error {
logger.DebugContext(ctx, fmt.Sprintf("testsRun: %+v", params))
args, total, err := testArgs("", params.ShardIndex, params.ShardTotal, params.Run, params.Skip, logger)
if err != nil {
return lazyerrors.Error(err)
}
args = append(args, params.Args...)
ot, err := observability.NewOTelTraceExporter(&observability.OTelTraceExporterOpts{
Logger: logger,
Service: "envtool-tests",
URL: "http://127.0.0.1:4318/v1/traces",
Service: "envtool-tests",
})
if err != nil {
return lazyerrors.Error(err)
}
ctx, span := otel.Tracer("").Start(ctx, "envtool.testsRun")
ctx, cancel := context.WithCancel(ctx)
done := make(chan struct{})
@@ -503,10 +601,30 @@ func testsRun(ctx context.Context, params *TestsRunParams, logger *slog.Logger)
close(done)
}()
err = runGoTest(ctx, args, total, true, logger)
defer func() {
span.End()
cancel()
<-done
}()
cancel()
<-done
args, total, err := testArgs(ctx, "", params.ShardIndex, params.ShardTotal, params.Run, params.Skip, logger)
if err != nil {
return lazyerrors.Error(err)
}
if len(params.Args) > 0 && params.Args[0] == "--" {
params.Args = params.Args[1:]
}
args = append(args, params.Args...)
err = runGoTest(ctx, &runGoTestOpts{
args: args,
total: total,
times: true,
rawPrefix: params.RawPrefix,
logger: logger,
})
return err
}
+83 -56
View File
@@ -16,7 +16,6 @@ package main
import (
"bytes"
"context"
"log/slog"
"os/exec"
"regexp"
@@ -26,8 +25,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/FerretDB/FerretDB/internal/util/logging"
"github.com/FerretDB/FerretDB/internal/util/testutil"
"github.com/FerretDB/FerretDB/v2/internal/util/logging"
"github.com/FerretDB/FerretDB/v2/internal/util/testutil"
)
var (
@@ -73,6 +72,8 @@ func bufLogger() (*slog.Logger, *bytes.Buffer) {
Level: slog.LevelInfo,
RemoveTime: true,
RemoveSource: true,
// the logger handles messages from the `go test` command output
SkipChecks: true,
})
return slog.New(h), &buf
@@ -81,7 +82,9 @@ func bufLogger() (*slog.Logger, *bytes.Buffer) {
func TestTestArgs(t *testing.T) {
t.Parallel()
list, listErr := listTestFuncs("testdata", ".", testutil.Logger(t))
ctx := testutil.Ctx(t)
list, listErr := listTestFuncs(ctx, "testdata", ".", testutil.Logger(t))
require.NoError(t, listErr)
require.Equal(
t,
@@ -92,7 +95,7 @@ func TestTestArgs(t *testing.T) {
t.Run("All", func(t *testing.T) {
t.Parallel()
actual, total, err := testArgs("testdata", 0, 0, "", "", testutil.Logger(t))
actual, total, err := testArgs(ctx, "testdata", 0, 0, "", "", testutil.Logger(t))
require.NoError(t, err)
expected := []string{"-run=^(TestError1|TestError2|TestNormal1|TestNormal2|TestPanic1|TestSkip1|TestWithSubtest)$"}
assert.Equal(t, expected, actual)
@@ -102,7 +105,7 @@ func TestTestArgs(t *testing.T) {
t.Run("Run", func(t *testing.T) {
t.Parallel()
actual, total, err := testArgs("testdata", 0, 0, "(?i)Normal", "", testutil.Logger(t))
actual, total, err := testArgs(ctx, "testdata", 0, 0, "(?i)Normal", "", testutil.Logger(t))
require.NoError(t, err)
expected := []string{"-run=^(TestNormal1|TestNormal2)$"}
assert.Equal(t, expected, actual)
@@ -112,7 +115,7 @@ func TestTestArgs(t *testing.T) {
t.Run("Subtest", func(t *testing.T) {
t.Parallel()
actual, total, err := testArgs("testdata", 0, 0, "TestWithSubtest/Second", "", testutil.Logger(t))
actual, total, err := testArgs(ctx, "testdata", 0, 0, "TestWithSubtest/Second", "", testutil.Logger(t))
require.NoError(t, err)
expected := []string{"-run=TestWithSubtest/Second"}
assert.Equal(t, expected, actual)
@@ -122,7 +125,7 @@ func TestTestArgs(t *testing.T) {
t.Run("Shard", func(t *testing.T) {
t.Parallel()
actual, total, err := testArgs("testdata", 1, 2, "", "", testutil.Logger(t))
actual, total, err := testArgs(ctx, "testdata", 1, 2, "", "", testutil.Logger(t))
require.NoError(t, err)
expected := []string{"-run=^(TestError1|TestNormal1|TestPanic1|TestWithSubtest)$"}
assert.Equal(t, expected, actual)
@@ -133,14 +136,18 @@ func TestTestArgs(t *testing.T) {
func TestRunGoTest(t *testing.T) {
t.Parallel()
ctx := context.TODO()
ctx := testutil.Ctx(t)
t.Run("Normal", func(t *testing.T) {
t.Parallel()
l, buf := bufLogger()
err := runGoTest(ctx, []string{"./testdata", "-count=1", "-run=TestNormal"}, 2, false, l)
err := runGoTest(ctx, &runGoTestOpts{
args: []string{"./testdata", "-count=1", "-run=TestNormal"},
total: 2,
logger: l,
})
require.NoError(t, err)
expected := []string{
@@ -148,8 +155,8 @@ func TestRunGoTest(t *testing.T) {
"INFO PASS TestNormal1 1/2",
"INFO PASS TestNormal2 2/2",
"INFO PASS",
"INFO ok github.com/FerretDB/FerretDB/cmd/envtool/testdata <SEC>s",
"INFO PASS github.com/FerretDB/FerretDB/cmd/envtool/testdata",
"INFO ok github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata <SEC>s",
"INFO PASS github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata",
}
actual := toLogLines(buf)
@@ -161,15 +168,19 @@ func TestRunGoTest(t *testing.T) {
l, buf := bufLogger()
err := runGoTest(ctx, []string{"./testdata", "-count=1", "-run=TestWithSubtest/Third"}, 1, false, l)
err := runGoTest(ctx, &runGoTestOpts{
args: []string{"./testdata", "-count=1", "-run=TestWithSubtest/Third"},
total: 1,
logger: l,
})
require.NoError(t, err)
expected := []string{
"INFO Running go test -json ./testdata -count=1 -run=TestWithSubtest/Third",
"INFO PASS TestWithSubtest 1/1",
"INFO PASS",
"INFO ok github.com/FerretDB/FerretDB/cmd/envtool/testdata <SEC>s",
"INFO PASS github.com/FerretDB/FerretDB/cmd/envtool/testdata",
"INFO ok github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata <SEC>s",
"INFO PASS github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata",
}
actual := toLogLines(buf)
@@ -181,7 +192,11 @@ func TestRunGoTest(t *testing.T) {
l, buf := bufLogger()
err := runGoTest(ctx, []string{"./testdata", "-count=1", "-run=TestWithSubtest/None"}, 1, false, l)
err := runGoTest(ctx, &runGoTestOpts{
args: []string{"./testdata", "-count=1", "-run=TestWithSubtest/None"},
total: 1,
logger: l,
})
require.NoError(t, err)
expected := []string{
@@ -189,8 +204,8 @@ func TestRunGoTest(t *testing.T) {
"INFO PASS TestWithSubtest 1/1",
"INFO testing: warning: no tests to run",
"INFO PASS",
"INFO ok github.com/FerretDB/FerretDB/cmd/envtool/testdata <SEC>s [no tests to run]",
"INFO PASS github.com/FerretDB/FerretDB/cmd/envtool/testdata",
"INFO ok github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata <SEC>s [no tests to run]",
"INFO PASS github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata",
}
actual := toLogLines(buf)
@@ -202,7 +217,11 @@ func TestRunGoTest(t *testing.T) {
l, buf := bufLogger()
err := runGoTest(ctx, []string{"./testdata", "-count=1", "-run=TestError"}, 2, false, l)
err := runGoTest(ctx, &runGoTestOpts{
args: []string{"./testdata", "-count=1", "-run=TestError"},
total: 2,
logger: l,
})
var exitErr *exec.ExitError
require.ErrorAs(t, err, &exitErr)
@@ -210,34 +229,34 @@ func TestRunGoTest(t *testing.T) {
expected := []string{
"INFO Running go test -json ./testdata -count=1 -run=TestError",
"WARN FAIL TestError1 1/2:",
"WARN === RUN TestError1",
"WARN error_test.go:20: not hidden 1",
"WARN error_test.go:22: Error 1",
"WARN error_test.go:24: not hidden 2",
"WARN --- FAIL: TestError1 (<SEC>s)",
"WARN ",
"WARN FAIL TestError2/Parallel:",
"WARN === RUN TestError2/Parallel",
"WARN error_test.go:35: not hidden 5",
"WARN === PAUSE TestError2/Parallel",
"WARN === CONT TestError2/Parallel",
"WARN error_test.go:39: not hidden 6",
"WARN error_test.go:41: Error 2",
"WARN error_test.go:43: not hidden 7",
"WARN --- FAIL: TestError2/Parallel (<SEC>s)",
"WARN ",
"WARN FAIL TestError2 2/2:",
"WARN === RUN TestError2",
"WARN error_test.go:28: not hidden 3",
"WARN === PAUSE TestError2",
"WARN === CONT TestError2",
"WARN error_test.go:32: not hidden 4",
"WARN --- FAIL: TestError2 (<SEC>s)",
"WARN ",
"ERROR FAIL TestError1 1/2:",
"ERROR === RUN TestError1",
"ERROR error_test.go:20: not hidden 1",
"ERROR error_test.go:22: Error 1",
"ERROR error_test.go:24: not hidden 2",
"ERROR --- FAIL: TestError1 (<SEC>s)",
"ERROR ",
"ERROR FAIL TestError2/Parallel:",
"ERROR === RUN TestError2/Parallel",
"ERROR error_test.go:35: not hidden 5",
"ERROR === PAUSE TestError2/Parallel",
"ERROR === CONT TestError2/Parallel",
"ERROR error_test.go:39: not hidden 6",
"ERROR error_test.go:41: Error 2",
"ERROR error_test.go:43: not hidden 7",
"ERROR --- FAIL: TestError2/Parallel (<SEC>s)",
"ERROR ",
"ERROR FAIL TestError2 2/2:",
"ERROR === RUN TestError2",
"ERROR error_test.go:28: not hidden 3",
"ERROR === PAUSE TestError2",
"ERROR === CONT TestError2",
"ERROR error_test.go:32: not hidden 4",
"ERROR --- FAIL: TestError2 (<SEC>s)",
"ERROR ",
"INFO FAIL",
"INFO FAIL github.com/FerretDB/FerretDB/cmd/envtool/testdata <SEC>s",
"INFO FAIL github.com/FerretDB/FerretDB/cmd/envtool/testdata",
"INFO FAIL github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata <SEC>s",
"INFO FAIL github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata",
}
actual := toLogLines(buf)
@@ -249,7 +268,11 @@ func TestRunGoTest(t *testing.T) {
l, buf := bufLogger()
err := runGoTest(ctx, []string{"./testdata", "-count=1", "-run=TestSkip"}, 1, false, l)
err := runGoTest(ctx, &runGoTestOpts{
args: []string{"./testdata", "-count=1", "-run=TestSkip"},
total: 1,
logger: l,
})
require.NoError(t, err)
expected := []string{
@@ -261,8 +284,8 @@ func TestRunGoTest(t *testing.T) {
"WARN --- SKIP: TestSkip1 (<SEC>s)",
"WARN ",
"INFO PASS",
"INFO ok github.com/FerretDB/FerretDB/cmd/envtool/testdata <SEC>s",
"INFO PASS github.com/FerretDB/FerretDB/cmd/envtool/testdata",
"INFO ok github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata <SEC>s",
"INFO PASS github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata",
}
actual := toLogLines(buf)
@@ -274,25 +297,29 @@ func TestRunGoTest(t *testing.T) {
l, buf := bufLogger()
err := runGoTest(ctx, []string{"./testdata", "-count=1", "-run=TestPanic"}, 1, false, l)
err := runGoTest(ctx, &runGoTestOpts{
args: []string{"./testdata", "-count=1", "-run=TestPanic"},
total: 1,
logger: l,
})
require.Error(t, err)
expected := []string{
"INFO Running go test -json ./testdata -count=1 -run=TestPanic",
"INFO FAIL github.com/FerretDB/FerretDB/cmd/envtool/testdata <SEC>s",
"INFO FAIL github.com/FerretDB/FerretDB/cmd/envtool/testdata",
"INFO FAIL github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata <SEC>s",
"INFO FAIL github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata",
"ERROR ",
"ERROR Some tests did not finish:",
"ERROR github.com/FerretDB/FerretDB/cmd/envtool/testdata.TestPanic1",
"ERROR github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata.TestPanic1",
"ERROR ",
"ERROR github.com/FerretDB/FerretDB/cmd/envtool/testdata.TestPanic1:",
"ERROR github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata.TestPanic1:",
"ERROR === RUN TestPanic1",
"ERROR panic: Panic 1",
"ERROR ",
"ERROR goroutine <ID> [running]:",
"ERROR github.com/FerretDB/FerretDB/cmd/envtool/testdata.TestPanic1.func1()",
"ERROR github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata.TestPanic1.func1()",
"ERROR <DIR>/cmd/envtool/testdata/panic_test.go:25 <ADDR>",
"ERROR created by github.com/FerretDB/FerretDB/cmd/envtool/testdata.TestPanic1 in goroutine <ID>",
"ERROR created by github.com/FerretDB/FerretDB/v2/cmd/envtool/testdata.TestPanic1 in goroutine <ID>",
"ERROR <DIR>/cmd/envtool/testdata/panic_test.go:23 <ADDR>",
"ERROR",
}
+211 -273
View File
@@ -12,16 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package main is the entry point for FerretDB server.
package main
import (
"context"
"encoding/json"
"expvar"
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"runtime"
runtimedebug "runtime/debug"
"strings"
"sync"
"sync/atomic"
@@ -30,161 +33,95 @@ import (
"github.com/alecthomas/kong"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/expfmt"
"go.uber.org/automaxprocs/maxprocs"
_ "golang.org/x/crypto/x509roots/fallback" // register root TLS certificates for production Docker image
"github.com/FerretDB/FerretDB/build/version"
"github.com/FerretDB/FerretDB/internal/clientconn"
"github.com/FerretDB/FerretDB/internal/clientconn/connmetrics"
"github.com/FerretDB/FerretDB/internal/handler/registry"
"github.com/FerretDB/FerretDB/internal/util/ctxutil"
"github.com/FerretDB/FerretDB/internal/util/debug"
"github.com/FerretDB/FerretDB/internal/util/debugbuild"
"github.com/FerretDB/FerretDB/internal/util/logging"
"github.com/FerretDB/FerretDB/internal/util/must"
"github.com/FerretDB/FerretDB/internal/util/observability"
"github.com/FerretDB/FerretDB/internal/util/password"
"github.com/FerretDB/FerretDB/internal/util/state"
"github.com/FerretDB/FerretDB/internal/util/telemetry"
"github.com/FerretDB/FerretDB/v2/build/version"
"github.com/FerretDB/FerretDB/v2/internal/clientconn"
"github.com/FerretDB/FerretDB/v2/internal/handler/middleware"
"github.com/FerretDB/FerretDB/v2/internal/util/ctxutil"
"github.com/FerretDB/FerretDB/v2/internal/util/debug"
"github.com/FerretDB/FerretDB/v2/internal/util/devbuild"
"github.com/FerretDB/FerretDB/v2/internal/util/iface"
"github.com/FerretDB/FerretDB/v2/internal/util/logging"
"github.com/FerretDB/FerretDB/v2/internal/util/must"
"github.com/FerretDB/FerretDB/v2/internal/util/observability"
"github.com/FerretDB/FerretDB/v2/internal/util/setup"
"github.com/FerretDB/FerretDB/v2/internal/util/state"
"github.com/FerretDB/FerretDB/v2/internal/util/telemetry"
)
// The cli struct represents all command-line commands, fields and flags.
// It's used for parsing the user input.
//
// Keep order in sync with documentation.
// Keep structure and order in sync with documentation and embeddable package.
//
//nolint:lll // for readability
var cli struct {
// We hide `run` command to show only `ping` in the help message.
Run struct{} `cmd:"" default:"1" hidden:""`
Ping struct{} `cmd:"" help:"Ping existing FerretDB instance."`
Version bool `default:"false" help:"Print version to stdout and exit." env:"-"`
Handler string `default:"postgresql" help:"${help_handler}"`
Mode string `default:"${default_mode}" help:"${help_mode}" enum:"${enum_mode}"`
StateDir string `default:"." help:"Process state directory."`
ReplSetName string `default:"" help:"Replica set name."`
Version bool `default:"false" help:"Print version to stdout and exit." env:"-"`
PostgreSQLURL string `name:"postgresql-url" default:"postgres://127.0.0.1:5432/postgres" help:"PostgreSQL URL." group:"PostgreSQL"`
PostgreSQLURLFile []byte `name:"postgresql-url-file" help:"Path to a file containing the PostgreSQL connection URL. If non-empty, this overrides --postgresql-url." group:"PostgreSQL" type:"filecontent"`
Listen struct {
Addr string `default:"127.0.0.1:27017" help:"Listen TCP address."`
Unix string `default:"" help:"Listen Unix domain socket path."`
TLS string `default:"" help:"Listen TLS address."`
Addr string `default:"127.0.0.1:27017" help:"Listen TCP address for MongoDB protocol."`
Unix string `default:"" help:"Listen Unix domain socket path for MongoDB protocol."`
TLS string `default:"" help:"Listen TLS address for MongoDB protocol."`
TLSCertFile string `default:"" help:"TLS cert file path."`
TLSKeyFile string `default:"" help:"TLS key file path."`
TLSCaFile string `default:"" help:"TLS CA file path."`
} `embed:"" prefix:"listen-"`
DataAPIAddr string `default:"" help:"Listen TCP address for HTTP Data API."`
MCPAddr string `default:"" help:"Listen TCP address for HTTP MCP server."`
} `embed:"" prefix:"listen-" group:"Interfaces"`
Proxy struct {
Addr string `default:"" help:"Proxy address."`
TLSCertFile string `default:"" help:"Proxy TLS cert file path."`
TLSKeyFile string `default:"" help:"Proxy TLS key file path."`
TLSCaFile string `default:"" help:"Proxy TLS CA file path."`
} `embed:"" prefix:"proxy-"`
} `embed:"" prefix:"proxy-" group:"Interfaces"`
DebugAddr string `default:"127.0.0.1:8088" help:"Listen address for HTTP handlers for metrics, profiling, etc."`
DebugAddr string `default:"127.0.0.1:8088" help:"Listen address for HTTP handlers for metrics, pprof, etc." group:"Interfaces"`
// see setCLIPlugins
kong.Plugins
Setup struct {
Database string `default:"" help:"Setup database during backend initialization."`
Username string `default:"" help:"Setup user during backend initialization."`
Password string `default:"" help:"Setup user's password."`
Timeout time.Duration `default:"30s" help:"Setup timeout."`
} `embed:"" prefix:"setup-"`
Mode string `default:"${default_mode}" help:"${help_mode}" enum:"${enum_mode}" group:"Miscellaneous"`
StateDir string `default:"." help:"Process state directory." group:"Miscellaneous"`
Auth bool `default:"true" help:"Enable authentication (on by default)." group:"Miscellaneous" negatable:""`
Log struct {
Level string `default:"${default_log_level}" help:"${help_log_level}"`
Format string `default:"console" help:"${help_log_format}" enum:"${enum_log_format}"`
UUID bool `default:"false" help:"Add instance UUID to all log messages." negatable:""`
} `embed:"" prefix:"log-"`
} `embed:"" prefix:"log-" group:"Miscellaneous"`
MetricsUUID bool `default:"false" help:"Add instance UUID to all metrics." negatable:""`
MetricsUUID bool `default:"false" help:"Add instance UUID to all metrics." group:"Miscellaneous" negatable:""`
OTel struct {
Traces struct {
ServiceName string `default:"ferretdb" help:"OpenTelemetry service name."`
Traces struct {
URL string `default:"" help:"OpenTelemetry OTLP/HTTP traces endpoint URL (e.g. 'http://host:4318/v1/traces')."`
} `embed:"" prefix:"traces-"`
} `embed:"" prefix:"otel-"`
} `embed:"" prefix:"otel-" group:"Miscellaneous"`
Telemetry telemetry.Flag `default:"undecided" help:"Enable or disable basic telemetry. See https://beacon.ferretdb.com."`
Telemetry telemetry.Flag `default:"undecided" help:"${help_telemetry}" group:"Miscellaneous"`
Test struct {
RecordsDir string `default:"" help:"Testing: directory for record files."`
DisablePushdown bool `default:"false" help:"Experimental: disable pushdown."`
EnableNestedPushdown bool `default:"false" help:"Experimental: enable pushdown for dot notation."`
CappedCleanup struct {
Interval time.Duration `default:"1m" help:"Experimental: capped collections cleanup interval."`
Percentage uint8 `default:"10" help:"Experimental: percentage of documents to cleanup."`
} `embed:"" prefix:"capped-cleanup-"`
EnableNewAuth bool `default:"false" help:"Experimental: enable new authentication."`
BatchSize int `default:"100" help:"Experimental: maximum insertion batch size."`
MaxBsonObjectSizeMiB int `default:"16" help:"Experimental: maximum BSON object size in MiB."`
Dev struct {
Version bool `hidden:""`
ReplSetName string `hidden:""`
RecordsDir string `hidden:""`
Telemetry struct {
URL string `default:"https://beacon.ferretdb.com/" help:"Telemetry: reporting URL."`
UndecidedDelay time.Duration `default:"1h" help:"Telemetry: delay for undecided state."`
ReportInterval time.Duration `default:"24h" help:"Telemetry: report interval."`
ReportTimeout time.Duration `default:"5s" help:"Telemetry: report timeout."`
Package string `default:"" help:"Telemetry: custom package type."`
URL string `default:"https://beacon.ferretdb.com/" hidden:""`
UndecidedDelay time.Duration `default:"1h" hidden:""`
ReportInterval time.Duration `default:"24h" hidden:""`
Package string `default:"" hidden:""`
} `embed:"" prefix:"telemetry-"`
} `embed:"" prefix:"test-"`
} `embed:"" prefix:"dev-"`
}
// The postgreSQLFlags struct represents flags that are used by the "postgresql" backend.
//
// See main_postgresql.go.
//
//nolint:lll // some tags are long
var postgreSQLFlags struct {
PostgreSQLURL string `name:"postgresql-url" default:"postgres://127.0.0.1:5432/ferretdb" help:"PostgreSQL URL for 'postgresql' handler."`
}
// The sqliteFlags struct represents flags that are used by the "sqlite" backend.
//
// See main_sqlite.go.
var sqliteFlags struct {
SQLiteURL string `name:"sqlite-url" default:"file:data/" help:"SQLite URI (directory) for 'sqlite' handler."`
}
// The mySQLFlags struct represents flags that are used by the "mysql" backend.
//
// See main_mysql.go.
var mySQLFlags struct {
MySQLURL string `name:"mysql-url" default:"mysql://127.0.0.1:3306/ferretdb" help:"MySQL URL for 'mysql' handler" hidden:""`
}
// The hanaFlags struct represents flags that are used by the "hana" handler.
//
// See main_hana.go.
var hanaFlags struct {
HANAURL string `name:"hana-url" help:"SAP HANA URL for 'hana' handler"`
}
// handlerFlags is a map of handler names to their flags.
var handlerFlags = map[string]any{}
// setCLIPlugins adds Kong flags for handlers in the right order.
func setCLIPlugins() {
handlers := registry.Handlers()
if len(handlers) != len(handlerFlags) {
panic("handlers and handlerFlags are not in sync")
}
for _, h := range handlers {
f := handlerFlags[h]
if f == nil {
panic(fmt.Sprintf("handler %q has no flags", h))
}
cli.Plugins = append(cli.Plugins, f)
}
}
// Additional variables for the kong parsers.
// Additional variables for [kong.Parse].
var (
logLevels = []string{
slog.LevelDebug.String(),
@@ -193,37 +130,34 @@ var (
slog.LevelError.String(),
}
logFormats = []string{"console", "text", "json"}
logFormats = []string{"console", "text", "json", "mongo"}
kongOptions = []kong.Option{
kong.HelpOptions{
Compact: true,
},
kong.Vars{
"default_log_level": defaultLogLevel().String(),
"default_mode": clientconn.AllModes[0],
"default_mode": middleware.AllModes[0],
"enum_log_format": strings.Join(logFormats, ","),
"enum_mode": strings.Join(clientconn.AllModes, ","),
"enum_mode": strings.Join(middleware.AllModes, ","),
"help_handler": fmt.Sprintf("Backend handler: '%s'.", strings.Join(registry.Handlers(), "', '")),
"help_log_format": fmt.Sprintf("Log format: '%s'.", strings.Join(logFormats, "', '")),
"help_log_level": fmt.Sprintf("Log level: '%s'.", strings.Join(logLevels, "', '")),
"help_mode": fmt.Sprintf("Operation mode: '%s'.", strings.Join(clientconn.AllModes, "', '")),
"help_mode": fmt.Sprintf("Operation mode: '%s'.", strings.Join(middleware.AllModes, "', '")),
"help_telemetry": "Enable or disable basic telemetry reporting. See https://beacon.ferretdb.com.",
},
kong.DefaultEnvars("FERRETDB"),
}
)
func main() {
setCLIPlugins()
ctx := kong.Parse(&cli, kongOptions...)
switch ctx.Command() {
case "run":
run()
case "ping":
logger := setupLogger(cli.Log.Format, "")
logger := setupDefaultLogger(cli.Log.Format, "")
checkFlags(logger)
ready := ReadyZ{
@@ -244,30 +178,34 @@ func main() {
// defaultLogLevel returns the default log level.
func defaultLogLevel() slog.Level {
if version.Get().DebugBuild {
if devbuild.Enabled {
return slog.LevelDebug
}
return slog.LevelInfo
}
// setupState setups state provider.
func setupState() *state.Provider {
var f string
// setupExpvar setups expvar variables for debug handler.
func setupExpvar(stateProvider *state.Provider) {
// do not include sensitive information like the full PostgreSQL URL
expvar.Publish("cli", iface.Stringer(func() string {
b := must.NotFail(json.Marshal(map[string]any{
"cli": map[string]any{
"log": map[string]any{
"level": cli.Log.Level,
},
},
}))
if dir := cli.StateDir; dir != "" && dir != "-" {
var err error
if f, err = filepath.Abs(filepath.Join(dir, "state.json")); err != nil {
log.Fatalf("Failed to get path for state file: %s.", err)
}
}
return string(b)
}))
sp, err := state.NewProvider(f)
if err != nil {
log.Fatal(stateFileProblem(f, err))
}
expvar.Publish("state", stateProvider.Var())
return sp
expvar.Publish("info", iface.Stringer(func() string {
b := must.NotFail(json.Marshal(version.Get()))
return string(b)
}))
}
// setupMetrics setups Prometheus metrics registerer with some metrics.
@@ -290,39 +228,59 @@ func setupMetrics(stateProvider *state.Provider) prometheus.Registerer {
return r
}
// setupLogger creates a logger with the level defined from cli.
func setupLogger(format string, uuid string) *slog.Logger {
// setupDefaultLogger setups slog logging.
func setupDefaultLogger(format string, uuid string) *slog.Logger {
var level slog.Level
if err := level.UnmarshalText([]byte(cli.Log.Level)); err != nil {
log.Fatal(err)
}
opts := &logging.NewHandlerOpts{
Base: format,
Level: level,
Base: format,
Level: level,
SkipChecks: !devbuild.Enabled,
}
logging.Setup(opts, uuid)
logging.SetupDefault(opts, uuid)
return slog.Default()
}
// checkFlags checks that CLI flags are not self-contradictory.
func checkFlags(l *slog.Logger) {
// checkFlags checks that CLI flags are not self-contradictory and produces warnings if needed.
// It also replaces "-" with "" for some flags to make it easier to configure FerretDB with environment variables.
func checkFlags(logger *slog.Logger) {
// keep in sync with documentation
for _, p := range []*string{
&cli.Listen.Addr,
&cli.Listen.Unix,
&cli.Listen.TLS,
&cli.Listen.DataAPIAddr,
&cli.Listen.MCPAddr,
&cli.DebugAddr,
&cli.OTel.Traces.URL,
} {
if *p == "-" {
*p = ""
}
}
ctx := context.Background()
if cli.Setup.Database != "" && !cli.Test.EnableNewAuth {
l.LogAttrs(ctx, logging.LevelFatal, "--setup-database requires --test-enable-new-auth")
if devbuild.Enabled {
logger.WarnContext(ctx, "This is a development build; the performance will be affected")
}
if (cli.Setup.Database == "") != (cli.Setup.Username == "") {
l.LogAttrs(ctx, logging.LevelFatal, "--setup-database should be used together with --setup-username")
if logger.Enabled(ctx, slog.LevelDebug) {
logger.WarnContext(ctx, "Debug logging is enabled; the performance will be affected")
}
if cli.Test.DisablePushdown && cli.Test.EnableNestedPushdown {
l.LogAttrs(
ctx,
logging.LevelFatal,
"--test-disable-pushdown and --test-enable-nested-pushdown should not be set at the same time",
if !cli.Auth {
logger.WarnContext(ctx, "Authentication is disabled; the server will accept any connection")
}
if cli.Auth && cli.Listen.MCPAddr != "" {
logger.Log(
ctx, logging.LevelFatal,
"MCP server does not support authentication; please disable authentication or MCP server",
)
}
}
@@ -339,7 +297,7 @@ func dumpMetrics() {
// run sets up environment based on provided flags and runs FerretDB.
func run() {
// to increase a chance of resource finalizers to spot problems
if debugbuild.Enabled {
if devbuild.Enabled {
defer func() {
runtime.GC()
runtime.GC()
@@ -348,17 +306,30 @@ func run() {
info := version.Get()
if p := cli.Test.Telemetry.Package; p != "" {
if cli.Dev.Version {
e := json.NewEncoder(os.Stdout)
e.SetIndent("", " ")
must.NoError(e.Encode(info))
buildInfo, ok := runtimedebug.ReadBuildInfo()
must.BeTrue(ok)
must.NoError(e.Encode(buildInfo))
return
}
if p := cli.Dev.Telemetry.Package; p != "" {
info.Package = p
}
if cli.Version {
fmt.Fprintln(os.Stdout, "version:", info.Version)
fmt.Fprintln(os.Stdout, "commit:", info.Commit)
fmt.Fprintln(os.Stdout, "branch:", info.Branch)
fmt.Fprintln(os.Stdout, "dirty:", info.Dirty)
fmt.Fprintln(os.Stdout, "package:", info.Package)
fmt.Fprintln(os.Stdout, "debugBuild:", info.DebugBuild)
_, _ = fmt.Fprintln(os.Stdout, "version:", info.Version)
_, _ = fmt.Fprintln(os.Stdout, "commit:", info.Commit)
_, _ = fmt.Fprintln(os.Stdout, "branch:", info.Branch)
_, _ = fmt.Fprintln(os.Stdout, "dirty:", info.Dirty)
_, _ = fmt.Fprintln(os.Stdout, "package:", info.Package)
_, _ = fmt.Fprintln(os.Stdout, "devBuild:", info.DevBuild)
_, _ = fmt.Fprintf(os.Stdout, "buildEnvironment: %+v\n", info.BuildEnvironment)
return
}
@@ -366,7 +337,16 @@ func run() {
// safe to always enable
runtime.SetBlockProfileRate(10000)
stateProvider := setupState()
if len(cli.PostgreSQLURLFile) > 0 {
cli.PostgreSQLURL = strings.TrimSpace(string(cli.PostgreSQLURLFile))
}
stateProvider, err := state.NewProviderDir(cli.StateDir)
if err != nil {
log.Fatalf("Failed to set up state provider: %s", err)
}
setupExpvar(stateProvider)
metricsRegisterer := setupMetrics(stateProvider)
@@ -376,7 +356,7 @@ func run() {
slog.String("branch", info.Branch),
slog.Bool("dirty", info.Dirty),
slog.String("package", info.Package),
slog.Bool("debugBuild", info.DebugBuild),
slog.Bool("devBuild", info.DevBuild),
slog.Any("buildEnvironment", info.BuildEnvironment),
}
logUUID := stateProvider.Get().UUID
@@ -387,31 +367,17 @@ func run() {
logUUID = ""
}
logger := setupLogger(cli.Log.Format, logUUID)
logger := setupDefaultLogger(cli.Log.Format, logUUID)
logger.LogAttrs(context.Background(), slog.LevelInfo, "Starting FerretDB "+info.Version+"...", startupFields...)
if debugbuild.Enabled {
logger.Info("This is a debug build. The performance will be affected.")
}
if logger.Enabled(context.Background(), slog.LevelDebug) {
logger.Info("Debug logging enabled. The security and performance will be affected.")
}
logger.LogAttrs(context.Background(), slog.LevelInfo, "Starting FerretDB "+info.Version, startupFields...)
checkFlags(logger)
if _, err := maxprocs.Set(maxprocs.Logger(func(format string, a ...any) {
logger.Debug(fmt.Sprintf(format, a...))
})); err != nil {
logger.Warn("Failed to set GOMAXPROCS", logging.Error(err))
}
ctx, stop := ctxutil.SigTerm(context.Background())
go func() {
<-ctx.Done()
logger.Info("Stopping...")
logger.InfoContext(ctx, "Stopping")
// second SIGTERM should immediately stop the process
stop()
@@ -422,7 +388,7 @@ func run() {
var wg sync.WaitGroup
if addr := cli.DebugAddr; addr != "" && addr != "-" {
if cli.DebugAddr != "" {
wg.Add(1)
go func() {
@@ -433,8 +399,8 @@ func run() {
l: l,
}
h, err := debug.Listen(&debug.ListenOpts{
TCPAddr: addr,
h, e := debug.Listen(&debug.ListenOpts{
TCPAddr: cli.DebugAddr,
L: l,
R: metricsRegisterer,
Livez: func(context.Context) bool {
@@ -444,17 +410,18 @@ func run() {
return listener.Load().Listening()
},
Readyz: ready.Probe,
})
if err != nil {
l.LogAttrs(ctx, logging.LevelFatal, "Failed to create debug handler", logging.Error(err))
if e != nil {
l.LogAttrs(ctx, logging.LevelFatal, "Failed to create debug handler", logging.Error(e))
}
h.Serve(ctx)
h.Run(ctx)
}()
}
if u := cli.OTel.Traces.URL; u != "" && u != "-" {
if cli.OTel.Traces.URL != "" {
wg.Add(1)
go func() {
@@ -462,120 +429,91 @@ func run() {
l := logging.WithName(logger, "otel")
ot, err := observability.NewOTelTraceExporter(&observability.OTelTraceExporterOpts{
ot, e := observability.NewOTelTraceExporter(&observability.OTelTraceExporterOpts{
Logger: l,
Service: "ferretdb",
Version: version.Get().Version,
URL: u,
URL: cli.OTel.Traces.URL,
Service: cli.OTel.ServiceName,
})
if err != nil {
l.LogAttrs(ctx, logging.LevelFatal, "Failed to create Otel tracer", logging.Error(err))
if e != nil {
l.LogAttrs(ctx, logging.LevelFatal, "Failed to create Otel tracer", logging.Error(e))
}
ot.Run(ctx)
}()
}
metrics := connmetrics.NewListenerMetrics()
mm := middleware.NewMetrics()
wg.Add(1)
{
wg.Add(1)
go func() {
defer wg.Done()
go func() {
defer wg.Done()
l := logging.WithName(logger, "telemetry")
opts := &telemetry.NewReporterOpts{
URL: cli.Test.Telemetry.URL,
F: &cli.Telemetry,
DNT: os.Getenv("DO_NOT_TRACK"),
ExecName: os.Args[0],
P: stateProvider,
ConnMetrics: metrics.ConnMetrics,
L: l,
UndecidedDelay: cli.Test.Telemetry.UndecidedDelay,
ReportInterval: cli.Test.Telemetry.ReportInterval,
ReportTimeout: cli.Test.Telemetry.ReportTimeout,
}
l := logging.WithName(logger, "telemetry")
r, err := telemetry.NewReporter(opts)
if err != nil {
l.LogAttrs(ctx, logging.LevelFatal, "Failed to create telemetry reporter", logging.Error(err))
}
tr, e := telemetry.NewReporter(&telemetry.NewReporterOpts{
URL: cli.Dev.Telemetry.URL,
Dir: cli.StateDir,
F: &cli.Telemetry,
DNT: os.Getenv("DO_NOT_TRACK"),
ExecName: os.Args[0],
P: stateProvider,
Metrics: mm,
L: l,
UndecidedDelay: cli.Dev.Telemetry.UndecidedDelay,
ReportInterval: cli.Dev.Telemetry.ReportInterval,
})
if e != nil {
l.LogAttrs(ctx, logging.LevelFatal, "Failed to create telemetry reporter", logging.Error(e))
}
r.Run(ctx)
}()
h, closeBackend, err := registry.NewHandler(cli.Handler, &registry.NewHandlerOpts{
Logger: logger,
ConnMetrics: metrics.ConnMetrics,
StateProvider: stateProvider,
TCPHost: cli.Listen.Addr,
ReplSetName: cli.ReplSetName,
SetupDatabase: cli.Setup.Database,
SetupUsername: cli.Setup.Username,
SetupPassword: password.WrapPassword(cli.Setup.Password),
SetupTimeout: cli.Setup.Timeout,
PostgreSQLURL: postgreSQLFlags.PostgreSQLURL,
SQLiteURL: sqliteFlags.SQLiteURL,
HANAURL: hanaFlags.HANAURL,
MySQLURL: mySQLFlags.MySQLURL,
TestOpts: registry.TestOpts{
DisablePushdown: cli.Test.DisablePushdown,
EnableNestedPushdown: cli.Test.EnableNestedPushdown,
CappedCleanupInterval: cli.Test.CappedCleanup.Interval,
CappedCleanupPercentage: cli.Test.CappedCleanup.Percentage,
EnableNewAuth: cli.Test.EnableNewAuth,
BatchSize: cli.Test.BatchSize,
MaxBsonObjectSizeBytes: cli.Test.MaxBsonObjectSizeMiB * 1024 * 1024,
},
})
if err != nil {
logger.LogAttrs(ctx, logging.LevelFatal, "Failed to construct handler", logging.Error(err))
tr.Run(ctx)
}()
}
defer closeBackend()
//exhaustruct:enforce
res := setup.Setup(ctx, &setup.SetupOpts{
Logger: logger,
StateProvider: stateProvider,
Metrics: mm,
l, err := clientconn.Listen(&clientconn.NewListenerOpts{
TCP: cli.Listen.Addr,
Unix: cli.Listen.Unix,
TLS: cli.Listen.TLS,
TLSCertFile: cli.Listen.TLSCertFile,
TLSKeyFile: cli.Listen.TLSKeyFile,
TLSCAFile: cli.Listen.TLSCaFile,
PostgreSQLURL: cli.PostgreSQLURL,
Auth: cli.Auth,
ReplSetName: cli.Dev.ReplSetName,
SessionCleanupInterval: 0,
ProxyAddr: cli.Proxy.Addr,
ProxyTLSCertFile: cli.Proxy.TLSCertFile,
ProxyTLSKeyFile: cli.Proxy.TLSKeyFile,
ProxyTLSCAFile: cli.Proxy.TLSCaFile,
Mode: clientconn.Mode(cli.Mode),
Metrics: metrics,
Handler: h,
Logger: logger,
TestRecordsDir: cli.Test.RecordsDir,
TCPAddr: cli.Listen.Addr,
UnixAddr: cli.Listen.Unix,
TLSAddr: cli.Listen.TLS,
TLSCertFile: cli.Listen.TLSCertFile,
TLSKeyFile: cli.Listen.TLSKeyFile,
TLSCAFile: cli.Listen.TLSCaFile,
Mode: middleware.Mode(cli.Mode),
TestRecordsDir: cli.Dev.RecordsDir,
DataAPIAddr: cli.Listen.DataAPIAddr,
MCPAddr: cli.Listen.MCPAddr,
})
if err != nil {
logger.LogAttrs(ctx, logging.LevelFatal, "Failed to construct listener", logging.Error(err))
if res == nil {
os.Exit(1)
}
listener.Store(l)
listener.Store(res.WireListener)
metricsRegisterer.MustRegister(l)
metricsRegisterer.MustRegister(res)
l.Run(ctx)
logger.Info("Listener stopped")
res.Run(ctx)
wg.Wait()
if info.DebugBuild {
if devbuild.Enabled {
dumpMetrics()
}
}
-22
View File
@@ -1,22 +0,0 @@
// Copyright 2021 FerretDB Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build ferretdb_hana
package main
// init adds "hana" handler flags when "ferretdb_hana" build tag is provided.
func init() {
handlerFlags["hana"] = &hanaFlags
}
-22
View File
@@ -1,22 +0,0 @@
// Copyright 2021 FerretDB Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !ferretdb_no_sqlite
package main
// init adds "sqlite" backend flags.
func init() {
handlerFlags["sqlite"] = &sqliteFlags
}
@@ -16,6 +16,7 @@ package main
import (
"context"
"encoding/json"
"os/exec"
"path/filepath"
"regexp"
@@ -25,7 +26,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/FerretDB/FerretDB/internal/util/testutil"
"github.com/FerretDB/FerretDB/v2/internal/util/testutil"
)
func TestVersion(t *testing.T) {
@@ -57,3 +58,16 @@ func TestVersion(t *testing.T) {
assert.Equal(t, commit[1], revision[1])
}
func TestDeps(t *testing.T) {
t.Parallel()
var res struct {
Deps []string `json:"Deps"`
}
b, err := exec.Command("go", "list", "-json").Output()
require.NoError(t, err)
require.NoError(t, json.Unmarshal(b, &res))
assert.NotContains(t, res.Deps, "testing", `package "testing" should not be imported by non-testing code`)
}
-127
View File
@@ -1,127 +0,0 @@
// Copyright 2021 FerretDB Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"log/slog"
"net"
"net/url"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"github.com/FerretDB/FerretDB/internal/util/logging"
)
// ReadyZ represents the Readiness probe, which is used to run `ping`
// command against the FerretDB instance specified by cli flags.
type ReadyZ struct {
l *slog.Logger
}
// Probe executes ping queries to open listeners, and returns true if they succeed.
// Any errors that occure are passed through ReadyZ.l listener.
//
// It is only executed if --setup-database flag is set.
func (ready *ReadyZ) Probe(ctx context.Context) bool {
l := ready.l
if cli.Setup.Database == "" {
l.InfoContext(ctx, "Setup database not specified - skipping ping")
return true
}
var urls []string
if cli.Listen.Addr != "" {
host, port, err := net.SplitHostPort(cli.Listen.Addr)
if err != nil {
l.ErrorContext(ctx, "Getting host and port failed", logging.Error(err))
return false
}
l.DebugContext(ctx, fmt.Sprintf("--listen-addr flag is set. Ping to %s will be performed", cli.Listen.Addr))
if host == "" {
host = "127.0.0.1"
l.DebugContext(ctx, fmt.Sprintf("Host not specified, defaulting to %s", host))
}
u := &url.URL{
Scheme: "mongodb",
Host: net.JoinHostPort(host, port),
Path: cli.Setup.Database,
User: url.UserPassword(cli.Setup.Username, cli.Setup.Password),
}
urls = append(urls, u.String())
}
if cli.Listen.TLS != "" {
// TODO https://github.com/FerretDB/FerretDB/issues/4427
l.WarnContext(ctx, "TLS ping is not implemented yet")
}
if cli.Listen.Unix != "" {
l.DebugContext(ctx, fmt.Sprintf("--listen-unix flag is set. Ping to %s will be performed", cli.Listen.Unix))
urls = append(urls, "mongodb://"+url.PathEscape(cli.Listen.Unix))
}
if len(urls) == 0 {
l.InfoContext(ctx, "Neither --listen-addr nor --listen-unix nor --listen-tls flags were specified - skipping ping")
return true
}
for _, u := range urls {
l.DebugContext(ctx, fmt.Sprintf("Pinging %s", u))
var cancel func()
ctx, cancel = context.WithTimeout(ctx, cli.Setup.Timeout)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(u))
if err != nil {
l.ErrorContext(ctx, "Connection failed", logging.Error(err))
return false
}
pingErr := client.Ping(ctx, nil)
// do not leave connection open when ping error causes os.Exit with Fatal
if err = client.Disconnect(ctx); err != nil {
l.ErrorContext(ctx, "Disconnect failed", logging.Error(err))
return false
}
if pingErr != nil {
l.ErrorContext(ctx, "Ping failed", logging.Error(pingErr))
return false
}
var uri *url.URL
if uri, err = url.Parse(u); err == nil {
u = uri.Redacted()
}
l.InfoContext(ctx, fmt.Sprintf("Ping to %s successful", u))
}
return true
}
Loaded 100 of 1209 files, more files were not shown because too many files have changed in this diff. Show more