diff --git a/glossy/src/gloss.typ b/glossy/src/gloss.typ
index d5d9e99..e570566 100644
--- a/glossy/src/gloss.typ
+++ b/glossy/src/gloss.typ
@@ -7,6 +7,7 @@
 #let __gloss_entries = state("__gloss_entries", (:))
 
 #let __gloss_label_prefix = "__gloss:"
+#let __gloss_first_use_counter_postfix = ":first-use-count"
 
 // Normalizes a dictionary entry by ensuring all required and optional keys exist
 // with appropriate default values.
@@ -129,11 +130,14 @@
 // Returns:
 //   none: Updates state as a side effect
 //
-#let __mark_term_used(key) = {
+#let __mark_term_used(key, count-as-first-use) = {
   counter(__gloss_label_prefix + key).step()
+  if count-as-first-use {
+    counter(__gloss_label_prefix + key + __gloss_first_use_counter_postfix).step()
+  }
 }
 
-// Queries the term usage
+// Queries whether the term is used ANYWHERE
 //
 // Parameters:
 //   key (string): The glossary entry key
@@ -142,12 +146,29 @@
 // Returns:
 //   boolean: If the entry is used above the location in the document
 //
-#let __is_term_used(key, location: none) = {
+#let __is_term_ever_used(key, location: none) = {
   let c = counter(__gloss_label_prefix + key)
   c = if location == none {c.final()} else {c.at(location)}
   c.at(0) > 0
 }
 
+// Queries whether the term has been "first used"
+//
+// "First used" is used to determine how to display the term, either "long
+// (short)" or just "(short)"
+//
+// Parameters:
+//   key (string): The glossary entry key
+//   location (location): The location in the document. Use none for end of document
+//
+// Returns:
+//   boolean: If the entry is used above the location in the document
+//
+#let __is_term_first_used(key, location: none) = {
+  let c = counter(__gloss_label_prefix + key + __gloss_first_use_counter_postfix)
+  c = if location == none {c.final()} else {c.at(location)}
+  c.at(0) > 0
+}
 
 // Determine if the glossary contains a visible entry
 //
@@ -216,8 +237,9 @@
   // ---------------------------------------------------------------------------
   // Manage term usage counting and determine if it's the first reference
   // ---------------------------------------------------------------------------
-  let is_first_use = not __is_term_used(key, location: here())
-  __mark_term_used(key)
+  let is_first_use = not __is_term_first_used(key, location: here())
+  let count-as-first-use = ("short" not in modifiers and "long" not in modifiers and "both" not in modifiers)
+  __mark_term_used(key, count-as-first-use)
 
   // ---------------------------------------------------------------------------
   // Helper Functions
@@ -554,7 +576,7 @@
   // Collect and organize entries by group
   let output = (:)
   let all_entries = __gloss_entries.final()
-  let all_used = all_entries.keys().filter(key => __is_term_used(key))
+  let all_used = all_entries.keys().filter(key => __is_term_ever_used(key))
 
   // Determine which groups to process
   let all_groups = all_entries
diff --git a/glossy/tests/abbreviated-syntax/ref/1.png b/glossy/tests/abbreviated-syntax/ref/1.png
index d99fd9e..0def5d0 100644
Binary files a/glossy/tests/abbreviated-syntax/ref/1.png and b/glossy/tests/abbreviated-syntax/ref/1.png differ
diff --git a/glossy/tests/demo/ref/1.png b/glossy/tests/demo/ref/1.png
index fe0c567..fa47282 100644
Binary files a/glossy/tests/demo/ref/1.png and b/glossy/tests/demo/ref/1.png differ
diff --git a/glossy/tests/format-term/ref/1.png b/glossy/tests/format-term/ref/1.png
index 24a7d77..bf265a6 100644
Binary files a/glossy/tests/format-term/ref/1.png and b/glossy/tests/format-term/ref/1.png differ
diff --git a/glossy/tests/refs-in-captions/ref/1.png b/glossy/tests/refs-in-captions/ref/1.png
new file mode 100644
index 0000000..34d3905
Binary files /dev/null and b/glossy/tests/refs-in-captions/ref/1.png differ
diff --git a/glossy/tests/refs-in-captions/test.typ b/glossy/tests/refs-in-captions/test.typ
new file mode 100644
index 0000000..7f2cc5e
--- /dev/null
+++ b/glossy/tests/refs-in-captions/test.typ
@@ -0,0 +1,82 @@
+#import "/lib.typ": *
+
+// Our test glossary
+#show: init-glossary.with((
+    ShortInHeader: "short used in header",
+    ShortInCaption: "short used in caption",
+    LongInHeader: "long used in header",
+    LongInCaption: "long used in caption",
+    BothInHeader: "both used in header",
+    BothInCaption: "both used in caption",
+    NormalInHeader: "unmodified use in header",
+    NormalInCaption: "unmodified use in caption",
+))
+
+// Configure styling
+
+#set page(numbering: "1")
+#set page(margin: 0.5in)
+#set page(height: auto)
+#set text(size: 10pt)
+
+// Outlines
+
+#outline(title:"Table of Contents", indent: auto)
+#outline(title: "List of Figures", target: figure.where(kind: image))
+#block(
+  height:7em,
+  glossary(title: "List of Abbreviations", theme: theme-twocol)
+)
+
+#line(length: 100%)
+
+// Doc Body
+
+// Use all the terms in headings
+#heading(level: 1, outlined: false, [Use terms in HEADERS])
+
+=== `@ShortInHeader:short` = "@ShortInHeader:short"
+=== `@LongInHeader:long` = "@LongInHeader:long"
+=== `@BothInHeader:both` = "@BothInHeader:both"
+=== `@NormalInHeader` = "@NormalInHeader"
+
+#line(length: 100%)
+
+// Use all the terms in captions
+#heading(level: 1, outlined: false, [Use terms in CAPTIONS])
+
+#figure([], caption: [`@ShortInCaption:short` = "@ShortInCaption:short"])
+#figure([], caption: [`@LongInCaption:long` = "@LongInCaption:long"])
+#figure([], caption: [`@BothInCaption:both` = "@BothInCaption:both"])
+#figure([], caption: [`@NormalInCaption` = "@NormalInCaption"])
+
+#line(length: 100%)
+
+// Now use the terms in the body to see if they handle "first use" correctly
+#heading(level: 1, outlined: false, [Use terms in BODY])
+
+First `@ShortInHeader` = "@ShortInHeader".
+Second `@ShortInHeader` = "@ShortInHeader"
+
+First `@LongInHeader` = "@LongInHeader".
+Second `@LongInHeader` = "@LongInHeader".
+
+First `@BothInHeader` = "@BothInHeader".
+Second `@BothInHeader` = "@BothInHeader".
+
+First `@NormalInHeader` = "@NormalInHeader".
+Second `@NormalInHeader` = "@NormalInHeader".
+
+#v(0.5em)
+
+First `@ShortInCaption` = "@ShortInCaption".
+Second `@ShortInCaption` = "@ShortInCaption"
+
+First `@LongInCaption` = "@LongInCaption".
+Second `@LongInCaption` = "@LongInCaption".
+
+First `@BothInCaption` = "@BothInCaption".
+Second `@BothInCaption` = "@BothInCaption".
+
+First `@NormalInCaption` = "@NormalInCaption".
+Second `@NormalInCaption` = "@NormalInCaption".